Montage - Revision history

imported>T1t0 at 03:12, 8 July 2018

2018-07-08T03:12:06Z

← Older revision		Revision as of 03:12, 8 July 2018
Line 4:		Line 4:

	* Note: fits files can be open with the program skycat		* Note: fits files can be open with the program skycat

	~~----~~

	== Compile Montage binaries ==		== Compile Montage binaries ==
Line 20:		Line 18:
	cd ..		cd ..
	</pre>		</pre>

	~~----~~

	== Run Montage over m101 galaxy data ==		== Run Montage over m101 galaxy data ==

	* From http://montage.ipac.caltech.edu/docs/m101tutorial.html		* From http://montage.ipac.caltech.edu/docs/m101tutorial.html

	~~----~~

	=== Prepare the test ===		=== Prepare the test ===
Line 36:		Line 30:
	PATH=../Montage_v3.3/bin:$PATH		PATH=../Montage_v3.3/bin:$PATH
	</pre>		</pre>

	~~----~~

	=== Prepare input data in 'fits' directory ===		=== Prepare input data in 'fits' directory ===
Line 45:		Line 37:
	cd input_dir		cd input_dir
	</pre>		</pre>

	~~----~~

	=== Download astronomic data ===		=== Download astronomic data ===
Line 69:		Line 59:
	gzip -d *.fits.gz		gzip -d *.fits.gz
	</pre>		</pre>

	~~----~~

	=== Create image definition file ".hdr" ===		=== Create image definition file ".hdr" ===
Line 95:		Line 83:
	cd ..		cd ..
	</pre>		</pre>

	~~----~~

	=== Reproject images ===		=== Reproject images ===
Line 122:		Line 108:
	mImgtbl reprojected_dir metadata_dir/reprojected.tbl		mImgtbl reprojected_dir metadata_dir/reprojected.tbl
	</pre>		</pre>

	~~----~~

	=== Generating first version of 'uncorrected' mosaic ===		=== Generating first version of 'uncorrected' mosaic ===
Line 132:		Line 116:
	mJPEG -gray output_dir/mosaic_uncorrected.fits 20% 99.98% loglog -out output_dir/mosaic_uncorrected.jpg		mJPEG -gray output_dir/mosaic_uncorrected.fits 20% 99.98% loglog -out output_dir/mosaic_uncorrected.jpg
	</pre>		</pre>

	~~----~~

	=== Smooth out the background levels ===		=== Smooth out the background levels ===
Line 153:		Line 135:
	mBgExec -p reprojected_dir metadata_dir/reprojected.tbl metadata_dir/background_corrections.tbl background_corrections_dir		mBgExec -p reprojected_dir metadata_dir/reprojected.tbl metadata_dir/background_corrections.tbl background_corrections_dir
	</pre>		</pre>

	~~----~~

	=== Correct mosaic ===		=== Correct mosaic ===
Line 163:		Line 143:
	cd ..		cd ..
	</pre>		</pre>

	~~----~~

	== Run Montage Benchmark (serial run) ==		== Run Montage Benchmark (serial run) ==

	~~----~~

	=== Get benchmark data ===		=== Get benchmark data ===
Line 175:		Line 151:
	wget http://sukhna.isi.edu/wf_benchmark/pegasus_wf_benchmark.tgz		wget http://sukhna.isi.edu/wf_benchmark/pegasus_wf_benchmark.tgz
	</pre>		</pre>

	~~----~~

	=== Get DAG python api ===		=== Get DAG python api ===
Line 185:		Line 159:
	touch Pegasus/__init__.py		touch Pegasus/__init__.py
	</pre>		</pre>

	~~----~~

	=== Create the file 'pegasus_dependencies_generator.py' with this content ===		=== Create the file 'pegasus_dependencies_generator.py' with this content ===
Line 317:		Line 289:

	</pre>		</pre>

	~~----~~

	=== Create the script 'pegasus_run.sh' to support the run ===		=== Create the script 'pegasus_run.sh' to support the run ===
Line 345:		Line 315:
	# End		# End
	</pre>		</pre>

	~~----~~

	=== Generate the list of commands and run them sequencially ===		=== Generate the list of commands and run them sequencially ===
Line 359:		Line 327:
	cd ..		cd ..
	</pre>		</pre>

	~~----~~

	== Generate a Pegasus workflow of m101 galaxy for 0.5x0.5 degree of area of the sky ==		== Generate a Pegasus workflow of m101 galaxy for 0.5x0.5 degree of area of the sky ==
Line 386:		Line 352:
	(nohup time sh ../pegasus_run.sh) &> serial-run.log &		(nohup time sh ../pegasus_run.sh) &> serial-run.log &
	</pre>		</pre>

	~~----~~

	== Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky ==		== Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky ==

imported>T1t0: /* Download astronomic data */

2015-10-10T00:28:53Z

Download astronomic data

← Older revision		Revision as of 00:28, 10 October 2015
Line 50:		Line 50:
	=== Download astronomic data ===		=== Download astronomic data ===

	* Go to 'http://irsa.ipac.caltech.edu/applications/2MASS/IM/batch.html'		* Go to ' http://irsa.ipac.caltech.edu/applications/2MASS/IM/batch.html '

	use the first form		use the first form

imported>T1t0: /* Generate a Pegasus workflow of m101 galaxy for 0.5x0.5 degree of area of the sky */

2015-10-08T05:11:09Z

Generate a Pegasus workflow of m101 galaxy for 0.5x0.5 degree of area of the sky

← Older revision		Revision as of 05:11, 8 October 2015
Line 384:		Line 384:
	cp dag_dir/cimages.tbl inputdata/cimages_20150928_002641_303.tbl		cp dag_dir/cimages.tbl inputdata/cimages_20150928_002641_303.tbl
	cp dag_dir/region.hdr inputdata/region_20150928_002641_303.hdr		cp dag_dir/region.hdr inputdata/region_20150928_002641_303.hdr
	time sh ../pegasus_run.sh		(nohup time sh ../pegasus_run.sh) &> serial-run.log &
	</pre>		</pre>

imported>T1t0: /* Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky */

2015-10-08T05:10:51Z

Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky

← Older revision		Revision as of 05:10, 8 October 2015
Line 431:		Line 431:
	cp dag_dir/slist.tbl inputdata/slist_20150928_011927_8886.tbl		cp dag_dir/slist.tbl inputdata/slist_20150928_011927_8886.tbl
	cp dag_dir/shrunken.hdr inputdata/shrunken_20150928_011927_8886.hdr		cp dag_dir/shrunken.hdr inputdata/shrunken_20150928_011927_8886.hdr
	time sh ../pegasus_run.sh		(nohup time sh ../pegasus_run.sh) &> serial-run.log &
	</pre>		</pre>

imported>T1t0: /* Create the file 'pegasus_dependencies_generator.py' with this content */

2015-10-04T20:50:00Z

Create the file 'pegasus_dependencies_generator.py' with this content

← Older revision		Revision as of 20:50, 4 October 2015
Line 193:		Line 193:

	"""		"""
	~~This runs~~ the ~~unit test suite for~~ the ~~DAX3 module~~		Extract the list of commands serially equivalents to the input Pegasus workflow, in the correct order.

	v 1.2		v 1.3
	"""		"""

imported>T1t0: /* Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky */

2015-09-30T05:35:26Z

Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky

← Older revision		Revision as of 05:35, 30 September 2015
Line 429:		Line 429:
	cp dag_dir/region_2_1.hdr inputdata/region_2_1_20150928_011927_8886.hdr		cp dag_dir/region_2_1.hdr inputdata/region_2_1_20150928_011927_8886.hdr
	cp dag_dir/region_2_2.hdr inputdata/region_2_2_20150928_011927_8886.hdr		cp dag_dir/region_2_2.hdr inputdata/region_2_2_20150928_011927_8886.hdr
	cp dag_dir/~~statfile~~.tbl inputdata/~~statfile_20150928_011927_8886~~.tbl		cp dag_dir/slist.tbl inputdata/slist_20150928_011927_8886.tbl
			cp dag_dir/shrunken.hdr inputdata/shrunken_20150928_011927_8886.hdr
	time sh ../pegasus_run.sh		time sh ../pegasus_run.sh
	</pre>		</pre>

imported>T1t0: /* Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky */

2015-09-30T04:06:40Z

Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky

← Older revision		Revision as of 04:06, 30 September 2015
Line 429:		Line 429:
	cp dag_dir/region_2_1.hdr inputdata/region_2_1_20150928_011927_8886.hdr		cp dag_dir/region_2_1.hdr inputdata/region_2_1_20150928_011927_8886.hdr
	cp dag_dir/region_2_2.hdr inputdata/region_2_2_20150928_011927_8886.hdr		cp dag_dir/region_2_2.hdr inputdata/region_2_2_20150928_011927_8886.hdr
			cp dag_dir/statfile.tbl inputdata/statfile_20150928_011927_8886.tbl
	time sh ../pegasus_run.sh		time sh ../pegasus_run.sh
	</pre>		</pre>

imported>T1t0: /* Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky */

2015-09-30T03:14:13Z

Generate a Pegasus workflow of m101 galaxy for 6x6 degree of area of the sky

@@ Line 411: / Line 411: @@
 cp dag_dir/statfile.tbl inputdata/statfile_20150928_011927_8886.tbl
 cp dag_dir/pimages.tbl inputdata/pimages_20150928_011927_8886.tbl
-cp dag_dir/cimages.tbl inputdata/cimages_20150928_011927_8886.tbl
+cp dag_dir/cimages_0_0.tbl inputdata/cimages_0_0_20150928_011927_8886.tbl
-cp dag_dir/region.hdr inputdata/region_20150928_011927_8886.hdr
+cp dag_dir/cimages_0_1.tbl inputdata/cimages_0_1_20150928_011927_8886.tbl
 time sh ../pegasus_run.sh
 </pre>

imported>T1t0: /* Create the file 'pegasus_dependencies_generator.py' with this content */

2015-09-30T01:41:24Z

Create the file 'pegasus_dependencies_generator.py' with this content

← Older revision		Revision as of 01:41, 30 September 2015
Line 191:		Line 191:

	<pre>		<pre>


	"""		"""
	This runs the unit test suite for the DAX3 module		This runs the unit test suite for the DAX3 module

	v 1.1		v 1.2
	"""		"""

Line 242:		Line 241:

	def __init__(self, input, output):		def __init__(self, input, output):
	self.~~done_jobs~~ = OrderedSet()		self.identified_jobs = OrderedSet()
	self.adag = Pegasus.DAX2.parse(input)		self.adag = Pegasus.DAX2.parse(input)
	self.output = open(output, "w")		self.output = open(output, "w")

	try:		try:
	# ~~write~~ all jobs in order of dependency		# save all jobs in order of dependency
	for job in self.adag.jobs:		for job in self.adag.jobs:

	dep_jobs = self.getDependencyOrderedJobs(job)		# if job is in the list, all dependencies are too
			if job not in self.identified_jobs:
			# retrieve dependency jobs
			dep_jobs = self.getDependencyOrderedJobs(job)

	for dep_job in dep_jobs:		# save dep_job to be writed if not was already done
			for dep_job in dep_jobs:
			if dep_job not in self.identified_jobs:
			self.identified_jobs.push(dep_job)

	if ~~dep_job~~ not in self.~~done_jobs~~:		# save the job itself
	self.~~writeJob~~(~~dep_job~~)		if job not in self.identified_jobs:
	self.~~done_jobs~~.~~push~~(~~dep_job~~)		self.identified_jobs.push(job)

			# write saved jobs
			for job in self.identified_jobs:
			self.writeJob(job)

	~~if job not in self.done_jobs:~~
	~~self.writeJob(dep_job)~~
	~~self.done_jobs.push(job)~~
	finally:		finally:
	self.output.close()		self.output.close()

	def getDependencyOrderedJobs(self, ~~job~~):		def getDependencyOrderedJobs(self, job_to_check):

	dep_jobs = OrderedSet()		dep_jobs = OrderedSet()

	if ~~job~~ in self.adag.lookup:		if job_to_check in self.adag.lookup:
	dependencies = self.adag.lookup[~~job~~]		dependencies = self.adag.lookup[job_to_check]

	for dep_job in dependencies.parents:		for dep_job in dependencies.parents:

	sub_dep_jobs = self.getDependencyOrderedJobs(dep_job)		sub_dep_jobs = self.getDependencyOrderedJobs(dep_job)

	for sub_dep_job in sub_dep_jobs:		for sub_dep_job in sub_dep_jobs:

	if sub_dep_job not in dep_jobs:		if sub_dep_job not in dep_jobs:
	dep_jobs.push(sub_dep_job)		dep_jobs.push(sub_dep_job)

	~~if job not in dep_jobs:~~
	~~dep_jobs.push(job)~~

	return dep_jobs		return dep_jobs
Line 313:		Line 314:
	# job = pg.adag.lookup.keys()[10]		# job = pg.adag.lookup.keys()[10]
	# print (pg.getDependencyOrderedJobs(job))		# print (pg.getDependencyOrderedJobs(job))


	</pre>		</pre>

imported>T1t0: /* Create the file 'pegasus_dependencies_generator.py' with this content */

2015-09-30T01:26:45Z

Create the file 'pegasus_dependencies_generator.py' with this content

← Older revision		Revision as of 01:26, 30 September 2015
Line 191:		Line 191:

	<pre>		<pre>


	"""		"""
	This runs the unit test suite for the DAX3 module		This runs the unit test suite for the DAX3 module

			v 1.1
	"""		"""

	import argparse		import argparse
	import Pegasus.DAX2		import Pegasus.DAX2


			class OrderedSet():

			def __init(self):
			self.list = list()
			self.set = set()

			def __init__(self, *data):
			self.list = list()
			self.set = set()

			for val in data:
			self.push(data)

			def __getitem__(self, index):
			return self.list[index]

			def __len__(self):
			return len(self.set)

			def push(self, value):
			if value not in self.set:
			self.list.append(value)
			self.set.add(value)

			def pop(self, value):
			if value in self.set:
			self.list.remove(value)
			self.set.remove(value)

			def has(self, value):
			return value in self.set

			def __contains__(self, value):
			return value in self.set


Line 203:		Line 242:

	def __init__(self, input, output):		def __init__(self, input, output):
	self.done_jobs = []		self.done_jobs = OrderedSet()
	self.adag = Pegasus.DAX2.parse(input)		self.adag = Pegasus.DAX2.parse(input)
	self.output = open(output, "w")		self.output = open(output, "w")
Line 217:		Line 256:
	if dep_job not in self.done_jobs:		if dep_job not in self.done_jobs:
	self.writeJob(dep_job)		self.writeJob(dep_job)
	self.done_jobs.~~append~~(dep_job)		self.done_jobs.push(dep_job)

	if job not in self.done_jobs:		if job not in self.done_jobs:
	self.writeJob(dep_job)		self.writeJob(dep_job)
	self.done_jobs.~~append~~(job)		self.done_jobs.push(job)
	finally:		finally:
	self.output.close()		self.output.close()
Line 227:		Line 266:
	def getDependencyOrderedJobs(self, job):		def getDependencyOrderedJobs(self, job):

	dep_jobs = []		dep_jobs = OrderedSet()

	if job in self.adag.lookup:		if job in self.adag.lookup:
Line 239:		Line 278:

	if sub_dep_job not in dep_jobs:		if sub_dep_job not in dep_jobs:
	dep_jobs.~~append~~(sub_dep_job)		dep_jobs.push(sub_dep_job)

	if job not in dep_jobs:		if job not in dep_jobs:
	dep_jobs.~~append~~(job)		dep_jobs.push(job)

	return dep_jobs		return dep_jobs
Line 248:		Line 287:
	def writeJob(self, job):		def writeJob(self, job):

	out = []		out = OrderedSet()
	out.~~append~~(job.name)		out.push(job.name)

	for arg in job.arguments:		for arg in job.arguments:
	if isinstance(arg, str) or isinstance(arg, unicode):		if isinstance(arg, str) or isinstance(arg, unicode):
	out.~~append~~(arg)		out.push(arg)
	elif isinstance(arg, Pegasus.DAX2.Filename):		elif isinstance(arg, Pegasus.DAX2.Filename):
	out.~~append~~(arg.filename)		out.push(arg.filename)
	else:		else:
	raise Exception("getting type " + str(type(arg)) + " : " + arg)		raise Exception("getting type " + str(type(arg)) + " : " + arg)
Line 274:		Line 313:
	# job = pg.adag.lookup.keys()[10]		# job = pg.adag.lookup.keys()[10]
	# print (pg.getDependencyOrderedJobs(job))		# print (pg.getDependencyOrderedJobs(job))

	</pre>		</pre>