CIC Show-and-Tell: Nextflow

@robsyme

		  #!/usr/bin/env nextflow

		  cheers = Channel.from "Bonjour", "Γεια σου", "Ciao", "Hello", "Hola"

		  process sayHello {
		    input:
		    val x from cheers

		    """
		    echo $x world!
		    """
		  }
 
nextflow run main.nf
N E X T F L O W  ~  version 0.17.3
Launching ./main.nf'
[warm up] executor > local
[5a/097b61] Submitted process > sayHello (1)
[4a/34a4b4] Submitted process > sayHello (4)
[7f/81dfd5] Submitted process > sayHello (5)
[eb/8b1c4a] Submitted process > sayHello (3)
[69/5b3dc6] Submitted process > sayHello (2)
Hola world!
Hello world!
Ciao world!
Bonjour world!
Γεια σου world!

.
└── main.nf

.
├── main.nf
├── .nextflow.history
├── .nextflow.log
└── work
    ├── 4a
    │   └── 34a4b4d888d62c90ae71f7d5c8356f
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    ├── 5a
    │   └── 097b610d29c05984571e4562041e48
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    ├── 69
    │   └── 5b3dc6d2ad4e1028d25588d3f046b8
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    ├── 7f
    │   └── 81dfd57542e3a849f75dbb3af6a952
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    └── eb
        └── 8b1c4a81f1e4f05d445082f06753d9
            ├── .command.begin
            ├── .command.err
            ├── .command.out
            ├── .command.run
            ├── .command.sh
            └── .exitcode

.
├── main.nf
├── .nextflow.history
├── .nextflow.log
└── work
    ├── 4a
    │   └── 34a4b4d888d62c90ae71f7d5c8356f
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    ├── 5a
    │   └── 097b610d29c05984571e4562041e48
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    ├── 69
    │   └── 5b3dc6d2ad4e1028d25588d3f046b8
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    ├── 7f
    │   └── 81dfd57542e3a849f75dbb3af6a952
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    └── eb
        └── 8b1c4a81f1e4f05d445082f06753d9
            ├── .command.begin
            ├── .command.err
            ├── .command.out
            ├── .command.run
            ├── .command.sh
            └── .exitcode

cat work/4a/*/.command.run
#!/bin/bash
# NEXTFLOW TASK: sayHello (4)
set -e
set -u
NXF_DEBUG=${NXF_DEBUG:=0}; [[ $NXF_DEBUG > 2 ]] && set -x
<snip>
COUT=$PWD/.command.po; mkfifo "$COUT"
CERR=$PWD/.command.pe; mkfifo "$CERR"
tee .command.out < "$COUT" &
tee1=$!
tee .command.err < "$CERR" >&2 &
tee2=$!
(
/bin/bash -ue /home/demo/work/4a/34a4b.../.command.sh
) >"$COUT" 2<"$CERR" &
pid=$!
wait $pid || ret=$?
wait $tee1 $tee2
cat work/4a/*/.command.sh
#!/bin/bash -ue
echo 'Hola world!'
 
nextflow run -process.executor sge main.nf
N E X T F L O W  ~  version 0.17.3
Launching ./main.nf'
[warm up] executor > sge
[58/9ae571] Submitted process > sayHello (1)
[67/3517f1] Submitted process > sayHello (4)
[d9/daf56a] Submitted process > sayHello (2)
[8c/8472b0] Submitted process > sayHello (5)
[90/90b7be] Submitted process > sayHello (3)
Hola world!
Hello world!
Γεια σου world!
Ciao world!
Bonjour world!

.
├── main.nf
├── .nextflow.history
├── .nextflow.log
└── work
    ├── 58
    │   └── 9ae5711ebe2548f8bf3598ed351d84
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    ├── 67
    │   └── 3517f1fbfb9c06579f5971f2a6362a
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    ├── 8c
    │   └── 8472b0c97003479207f4c8bba77e54
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    ├── 90
    │   └── 90b7be0324f9bdc8613f290d01e18a
    │       ├── .command.begin
    │       ├── .command.err
    │       ├── .command.out
    │       ├── .command.run
    │       ├── .command.sh
    │       └── .exitcode
    └── d9
        └── daf56a586b3c6c108b4b7ab1a3dd2e
            ├── .command.begin
            ├── .command.err
            ├── .command.out
            ├── .command.run
            ├── .command.sh
            └── .exitcode

# After running with SGE executor
cat work/58/*/.command.sh
#!/bin/bash -ue
echo 'Hola world!'
head work/58/*/.command.run 
#!/bin/bash
#$ -wd /home/demo/work/58/9ae5711ebe2548f8bf3598ed351d84
#$ -N nf-sayHello_(1)
#$ -o /home/demo/work/58/9ae5711ebe2548f8bf3598ed351d84/.command.log
#$ -j y
#$ -terse
#$ -V
#$ -notify

	  
# NEXTFLOW TASK: sayHello (1)

	
      


      

	
Current Executors

	
      


      

	
	  
# On each worker node:

	  
nextflow node -bg

	  
Configuring Apache Ignite cluster daemon

	  

	  

	  

	  
# On the master node:

	  
nextflow run -process.executor ignite main.nf

	  
N E X T F L O W  ~  version 0.17.3

	  
Launching ./main.nf'

	  
[warm up] executor > ignite

	  
etc, etc

	
      


      

	
Joining Nextflow processes

      

      

      

	
	  
	    cheers = Channel.from "Bonjour", "Γεια σου", "Ciao", "Hello", "Hola"

	    process sayHello {
	      input:
	      val x from cheers
	    
	      output:
	      file 'output.txt' into greetings
	    
	      "echo '$x world' > output.txt"
	    }

	    process enthusiasm {
	      input:
	      file 'boring_greeting.txt' from greetings
	    
	      output:
	      file 'enthusiastic.txt' into friendly
	    
	      """
	      #!/usr/bin/env ruby
	      File.open('enthusiastic.txt', 'w') do |out|
	      out.print File.read('boring_greeting.txt').chomp
	      out.puts '!!!!'
	      end
	      """
	    }

	
      


      

	

.

├── main.nf

└── work

    ├── 09

    │   └── a1209973ec03499042bb693cde5373

    │       ├── boring_greeting.txt -> /home/demo/work/b5/93909.../output.txt

    │       └── enthusiastic.txt

    ├── 11

    │   └── a610f0c17e293224da640908fe0138

    │       └── output.txt

    ├── 12

    │   └── f67333524f72e464351306eeca6ca7

    │       ├── boring_greeting.txt -> /home/demo/work/2a/bcadc.../output.txt

    │       └── enthusiastic.txt

    ├── 2a

    │   └── bcadc505ed8865d34f4f4a5f4ccda6

    │       └── output.txt

    ├── 49

    │   └── 2e619875bb46ba2df926246cef961a

    │       ├── boring_greeting.txt -> /home/demo/work/11/a610f.../output.txt

    │       └── enthusiastic.txt

    ├── 95

    │   └── 65b1f6fb7f1cbb1faffe9e6dd93d9b

    │       └── output.txt

    ├── 99

    │   └── 6c3c7ccd08abe2b01bbccae0d31e03

    │       ├── boring_greeting.txt -> /home/demo/work/dc/c1a4c.../output.txt

    │       └── enthusiastic.txt

    ├── b5

    │   └── 9390903c9534a3d1e12707a0e281ec

    │       └── output.txt

    ├── dc

    │   └── c1a4c0b45df57dff6b99a377373236

    │       └── output.txt

    └── ec

        └── 6ec84ac25fe9ccb411f161e85c504c

            ├── boring_greeting.txt -> /home/demo/work/95/65b1f6.../output.txt

            └── enthusiastic.txt

      


      

	
Livecoding time


	
(a.k.a. The slides are too small to fit the code)

      


      

        
Thanks, CIC!

	http://nextflow.io
        
@robsyme