Jump to navigation Jump to search


  • Doc v1.3
  • Note: fits files can be open with the program skycat

Compile Montage binaries

  • Note: remember to install 'libc6-dev-amd64'
  • Note: for Montage_v3.0, remember to add '-lpthread' and '-ldl' options in Montage_v3.0/grid/Pegasus/Makefile.LINUX
tar -xzf Montage_v3.3.tar.gz
cd Montage_v3.3
cd ..

Run Montage over m101 galaxy data

Prepare the test

mkdir test_m101_from_irsa
cd test_m101_from_irsa

Prepare input data in 'fits' directory

mkdir input_dir
cd input_dir

Download astronomic data

use the first form

>   Retrieve 'ATLAS' images 'overlapping' a Region
>   All Sky Release Survey Images
>   Region Center =                 M 101
>   Region Diameter =               0.5 // original tutorial is a rectangle of 0.2 degree area, sqr(0.2) = 0.447
>   Decompress Quicklook images  =  false // less data to download
>   Band =                          J // in the montage benckmark file the J band is the only defined
  • Save and extract the archives
tar -xf m101_0.5degree_j.tar
gzip -d *.fits.gz

Create image definition file ".hdr"


>   Object or location:     M 101
>   System:                 Equatorial
>   Equinox:                2000.
>   Width:                  0.5
>   Height:                 0.5
>   Resolution:             1.0
>   Rotation:               0.0
  • Write the result into 'template.hdr' file
  • Go back to the base directory
cd ..

Reproject images

mkdir metadata_dir
  • Create a .tbl file describing your input data
mImgtbl input_dir metadata_dir/input_fits.tbl
  • Execute reprojecting
mkdir reprojected_dir
mProjExec -p input_dir metadata_dir/input_fits.tbl input_dir/template.hdr reprojected_dir metadata_dir/reprojected_stats.tbl
  • Recreate a .tbl file describing your generated reprojected data
mImgtbl reprojected_dir metadata_dir/reprojected.tbl

Generating first version of 'uncorrected' mosaic

mkdir output_dir
mAdd -p reprojected_dir metadata_dir/reprojected.tbl input_dir/template.hdr output_dir/mosaic_uncorrected.fits
mJPEG -gray output_dir/mosaic_uncorrected.fits 20% 99.98% loglog -out output_dir/mosaic_uncorrected.jpg

Smooth out the background levels

  • Calculate the best way to smooth out the overlap regions using the difference images
mOverlaps metadata_dir/reprojected.tbl metadata_dir/diffs.tbl
mkdir diff_dir
mDiffExec -p reprojected_dir metadata_dir/diffs.tbl input_dir/template.hdr diff_dir
mFitExec metadata_dir/diffs.tbl metadata_dir/plane_fitting.tbl diff_dir
  • Apply the background removal to the original reprojected images
mBgModel metadata_dir/reprojected.tbl metadata_dir/plane_fitting.tbl metadata_dir/background_corrections.tbl
mkdir background_corrections_dir
mBgExec -p reprojected_dir metadata_dir/reprojected.tbl metadata_dir/background_corrections.tbl background_corrections_dir

Correct mosaic

mAdd -p background_corrections_dir metadata_dir/reprojected.tbl input_dir/template.hdr output_dir/mosaic_corrected.fits
mJPEG -gray output_dir/mosaic_corrected.fits 0s max gaussian-log -out output_dir/mosaic_corrected.jpg
cd ..

Run Montage Benchmark (serial run)

Get benchmark data


Get DAG python api

mkdir Pegasus
wget -P Pegasus
touch Pegasus/

Create the file '' with this content

Extract the list of commands serially equivalents to the input Pegasus workflow, in the correct order.

v 1.3

import argparse
import Pegasus.DAX2

class OrderedSet():

    def __init(self):
        self.list = list()
        self.set = set()

    def __init__(self, *data):
        self.list = list()
        self.set = set()

        for val in data:

    def __getitem__(self, index):
        return self.list[index]

    def __len__(self):
        return len(self.set)

    def push(self, value):
        if value not in self.set:

    def pop(self, value):
        if value in self.set:

    def has(self, value):
        return value in self.set

    def __contains__(self, value):
        return value in self.set

class PegasusDependenciesGenerator:

    def __init__(self, input, output):
        self.identified_jobs = OrderedSet()
        self.adag = Pegasus.DAX2.parse(input)
        self.output = open(output, "w")

            # save all jobs in order of dependency
            for job in

                # if job is in the list, all dependencies are too
                if job not in self.identified_jobs:
                    # retrieve dependency jobs
                    dep_jobs = self.getDependencyOrderedJobs(job)

                    # save dep_job to be writed if not was already done
                    for dep_job in dep_jobs:
                        if dep_job not in self.identified_jobs:

                    # save the job itself
                    if job not in self.identified_jobs:

            # write saved jobs
            for job in self.identified_jobs:


    def getDependencyOrderedJobs(self, job_to_check):

        dep_jobs = OrderedSet()

        if job_to_check in self.adag.lookup:
            dependencies = self.adag.lookup[job_to_check]

            for dep_job in dependencies.parents:
                sub_dep_jobs = self.getDependencyOrderedJobs(dep_job)

                for sub_dep_job in sub_dep_jobs:
                    if sub_dep_job not in dep_jobs:

        return dep_jobs

    def writeJob(self, job):

        out = OrderedSet()

        for arg in job.arguments:
            if isinstance(arg, str) or isinstance(arg, unicode):
            elif isinstance(arg, Pegasus.DAX2.Filename):
                raise Exception("getting type " + str(type(arg)) + " : " + arg)

        self.output.write(" ".join(out) + "\n")

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
        "input", type=str, help="the input dax XML file")
        "output", type=str, help="the output dependencies file")
    args = parser.parse_args()

    pg = PegasusDependenciesGenerator(args.input, args.output)
    # job = pg.adag.lookup.keys()[10]
    # print (pg.getDependencyOrderedJobs(job))

Create the script '' to support the run



test ! -d ${INPUT} && echo >&2 "Error: no '${INPUT}' folder, exit" && exit 1;
test ! -d ${BIN} && echo >&2 "Error: no '${BIN}' folder, exit" && exit 1;
test ! -f ${COMMANDS} && echo >&2 "Error: no '${COMMANDS}' file, exit" && exit 1;


cd ${INPUT};
cat ../${COMMANDS} | while read CMD; do
    /bin/echo -e "${CMD}\t...";
    test "$?" -ne 0 && echo "Error, exit" && exit 1;
    /bin/echo -e "Done.";

# End

Generate the list of commands and run them sequencially

mkdir test_montage_benchmark
cd test_montage_benchmark
tar -xzf ../pegasus_wf_benchmark.tgz
python ../ pegasus_wf_benchmark/montage/montage-1.0-dax/montage-1.0.dax dependencies.txt
tar -xzf pegasus_wf_benchmark/montage/input_data.tgz
time sh ../
cd ..

Generate a Pegasus workflow of m101 galaxy for 0.5x0.5 degree of area of the sky

mkdir test_m101_benchmark_0.5x0.5
cd test_m101_benchmark_0.5x0.5
mkdir dag_dir
mDAG 2mass j M101 0.5 0.5 0.000278 dag_dir "gsi`pwd`" "gsi`pwd`/input"
mkdir inputdata
cd inputdata
mArchiveExec ../dag_dir/images.tbl
gzip -d *.fits.gz
cd ..
test -f inputdata.tgz || tar -czf inputdata.tgz inputdata  # save a backup of downloaded data
python ../ dag_dir/dag.xml dependencies.txt
cp dag_dir/big_region.hdr inputdata/big_region_20150928_002641_303.hdr
cp dag_dir/statfile.tbl inputdata/statfile_20150928_002641_303.tbl
cp dag_dir/pimages.tbl inputdata/pimages_20150928_002641_303.tbl
cp dag_dir/cimages.tbl inputdata/cimages_20150928_002641_303.tbl
cp dag_dir/region.hdr inputdata/region_20150928_002641_303.hdr
(nohup time sh ../ &> serial-run.log &

Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky

mkdir test_m101_benchmark_6x6
cd test_m101_benchmark_6x6
mkdir dag_dir
mDAG 2mass j M101 6 6 0.000278 dag_dir "gsi`pwd`" "gsi`pwd`/input"
mkdir inputdata
cd inputdata
mArchiveExec -d 1 ../dag_dir/images.tbl
cd ..
test -f inputdata.tar || tar -cf inputdata.tar inputdata  # save a backup of downloaded data
cd inputdata
gzip -d *.fits.gz
cd ..
python ../ dag_dir/dag.xml dependencies.txt
cp dag_dir/big_region.hdr inputdata/big_region_20150928_011927_8886.hdr
cp dag_dir/statfile.tbl inputdata/statfile_20150928_011927_8886.tbl
cp dag_dir/pimages.tbl inputdata/pimages_20150928_011927_8886.tbl
cp dag_dir/cimages_0_0.tbl inputdata/cimages_0_0_20150928_011927_8886.tbl
cp dag_dir/cimages_0_1.tbl inputdata/cimages_0_1_20150928_011927_8886.tbl
cp dag_dir/cimages_0_2.tbl inputdata/cimages_0_2_20150928_011927_8886.tbl
cp dag_dir/cimages_1_0.tbl inputdata/cimages_1_0_20150928_011927_8886.tbl
cp dag_dir/cimages_1_1.tbl inputdata/cimages_1_1_20150928_011927_8886.tbl
cp dag_dir/cimages_1_2.tbl inputdata/cimages_1_2_20150928_011927_8886.tbl
cp dag_dir/cimages_2_0.tbl inputdata/cimages_2_0_20150928_011927_8886.tbl
cp dag_dir/cimages_2_1.tbl inputdata/cimages_2_1_20150928_011927_8886.tbl
cp dag_dir/cimages_2_2.tbl inputdata/cimages_2_2_20150928_011927_8886.tbl
cp dag_dir/region_0_0.hdr inputdata/region_0_0_20150928_011927_8886.hdr
cp dag_dir/region_0_1.hdr inputdata/region_0_1_20150928_011927_8886.hdr
cp dag_dir/region_0_2.hdr inputdata/region_0_2_20150928_011927_8886.hdr
cp dag_dir/region_1_0.hdr inputdata/region_1_0_20150928_011927_8886.hdr
cp dag_dir/region_1_1.hdr inputdata/region_1_1_20150928_011927_8886.hdr
cp dag_dir/region_1_2.hdr inputdata/region_1_2_20150928_011927_8886.hdr
cp dag_dir/region_2_0.hdr inputdata/region_2_0_20150928_011927_8886.hdr
cp dag_dir/region_2_1.hdr inputdata/region_2_1_20150928_011927_8886.hdr
cp dag_dir/region_2_2.hdr inputdata/region_2_2_20150928_011927_8886.hdr
cp dag_dir/slist.tbl inputdata/slist_20150928_011927_8886.tbl
cp dag_dir/shrunken.hdr inputdata/shrunken_20150928_011927_8886.hdr
(nohup time sh ../ &> serial-run.log &