#!/bin/csh # NOTE requires the C/T shell to run properly. However, one could create a # global parameter file that defines the shell run command (i.e., # EXECUTE="source" for C/T shell and EXECUTE="." for Bash) and provide this # in isisminer's GLOBAL parameter. Then include EXECUTE in the CommandArgs # keyword and its %n position in that keyword in the Matchem command in the # MultiImageMatcher/SingleImageMatcher Strategies that runs the findfeatures # commands. ############################################################################### # Prior steps to application of this script is to identify all images in # a 10x10 degree region, for each image in each region determine all images # that overlap it. There is another step that computes these input files # using caminfo input. We used a database, the PVL files can be used as well. # The result is a CSV file that contains many fields of data containing a list # of every image that overlaps it. This file should be provided as the FROM # parameter in isisnminer. # # This isisminer CONFIG file computes image-to-image control networks using # feature-based matching of overlapping files. This config is designed to # process all MDIS WAC images only. Limb images are identified by the # observation type and excluded from processing. There are no other # restrictions. # # High level processing is designed around the 10x10 degree regions that # contain images whose center latitude and longitude coordinate fall within # the region. Overlaps for all images are computed for the region images # without restriction - i.e., all images that overlap individual region # images are identified for matching. Currently, any image that has 3.5% # of common overlap with one another is included for matching. # # This config takes a single overlap region file stored in CSV format. This # file was created by an earlier run of isisminer that considered all orbital # images as described above. It will filter out all NAC and limb image # observations and execute the findfeatures application to simulatenously # (using threads!) match all overlapping images using OpenCV feature matching # algorithms. It creates a list of images that were not successfully # matched in multi-match mode (must have >= 8 points) and runs them as pairs # through findfeatures again with small adjustments to the paramters and has # a Scharr filter applied to both images to enhance edges for better matching # results. # # This processing results in a multi-match control and potentially many # singleton (single match pair) networks. These are combined into one final # network using the cnetcombinept app. All extraneous files can be deleted # by uncommenting the proper commands. # # Once all regions have been processed, all image-based networks are combined # into a single global network using cnetcombinept. Then jigsaw is ran to # complete the control. # # INPUTS # There are five required parameters to isisminer to execute this config. And # certain requirements are assumed to be contained in of the overlaping CSV # file. # # The description of the PARAMETERS required for this run are: # # multi_algorithms - contains the specification of the OpenCV feature # matching algorithm applied in multi-match mode. # # single_algorithms - contains the specification of the OpenCV feature # matching algorithm applied in singleton, pair-wise mode. # # imagedir - Specifies the root of the path to the MDIS image archive. # Example: /scratch/kbecker/MESSENGER/Lev1 # # from - The input CSV file containing match file on the first row after the # header and all subsequent rows contain images that overlap with # it. # Example: /scratch/kbecker/MESSENGER/Control/Overlaps/Equi025S305ED10/EW0210888537G_overlaps.csv # # outputdir - Path to write the image-based control networks. # Example: /scratch/kbecker/MESSENGER/Control/WACOnly # # The full command to execute a single file in a region is (for berevity, # let SCRIPTS=$ISIS3DATA/base/templates/isisminer and IN = /scratch/kbecker/MESSENGER): # isisminer config=$SCRIPTS/mdis_wac_matcher.conf # parameters="multi_algorithms:$SCRIPTS/multi_algorithms@ # single_algorithms:$SCRIPTS/single_algorithms.lis@ # imagedir:$IN/Lev1@ # from:$IN/Lev1@ # outputdir:$IN/Control/WACOnly" # # outputdir need not exist prior to processing but must be createable via # the "mkdir -p" command. Directories that will be created in the script are: # {outputdir}/{Region}_Networks - Region is of the form EquiLAT[S/N]LON[E/W]D10 # # In that directory will be the all the image-based networks for that 10x10 # region of images. The file that will be combined into the image-based # network are of the form Equi*_Networks/EW0210888537G_combined.net. # # @author 2016-02-18 Kris Becker # ############################################################################### Object = IsisMiner # This config is designed to work with overlap files generated using the # GisOverlap strategy that creates a file of all overlapping images. Name = MdisGlobalControl RequiredParameters = ( "multi_algorithms", "single_algorithms", "ImageDir", "From", "OutputDir" ) # Not applied but designed for these input files. The from paramter is a # single overlap file. If a NAC overlap file is given in from, it will be # ignored (see the SelectWACImages Filter Strategy). Object = Defaults Config = "$PWD/mdis_wac_matcher.conf" Multi_Algorithms = "$PWD/multi_algorithms.lis" Single_Algorithms = "$PWD/single_algorithms.lis" ImageDir = "/scratch/kbecker/MESSENGER/Lev1" From = "$PWD/Equi035N235ED10/*_overlaps.csv" OutputDir = "$PWD" EndObject # Reads overlap file where first line is match file, all other lines contain # data for overlaping files. Object = Strategy Name = ReadImageOverlaps Type = CsvReader Description = "Read regional overlap image set" CsvFileArgs = "From" CsvFile = "%1" # Reading the header sets the variable names for each column in each # row (Resource) HasHeader = True Identity = "%1" IdentityArgs = "SourceProductId" EndObject # This strategy eliminates all NAC images from the list (which could be # the match image). Object = Strategy # Select WAC images Name = SelectWACImages Type = Filter Description = "Use RegExp to find all WAC only images" Keyword = SourceProductId RegExp = "^EW." EndObject # Eliminate all limb observation images. Object = Strategy Name = NoLimbsPlease Type = Filter Description = "Eliminate all explicit limb observations..." Keyword = ObservationType Exclude = "Limb" EndObject # Completely remove all images, including MATCH image, if they did not pass # constraints in Limit::SelectWACImages and Filter::NoLimbsPlease. Object = Strategy Name = DeleteImagesNotPassingConstraints Type = ResourceManager Description = "Delete all images that do not pass the above constraints" Operations = ( DeleteDiscard ) EndObject # Find the FROMLIST image set that passed the constraints Object = Strategy Name = SelectMatchImage Type = IsNumeric Description = "(Non)Select MATCH image" Keyword = OverlapRatioMatch EndObject # See if we have a MATCH file. Won't do the AssetSidebar if it didn't pass # constraints meaning the match image didn't make the cut. Object = Strategy Name = ChooseMatchImage Type = ResourceManager Description = "Select MATCH image" Operations = ( ToggleDiscard ) EndObject # This asset sidebar puts the MATCH file in the global variable pool and # sets up environment to run the matcher with results set to FromList Object = Strategy Name = MultiMatcherLogic Type = AssetSidebar Asset = FromList Operation = Create CreateSource = Copy # Sidebar processing strategies. Toggle active/inactive get at match image # and train images. This multi-strategy object applies multi-image # matching. Object = Isisminer Name = MatcherLogic # Enable all remaining resources for overlap constraints testing Object = Strategy Name = ChooseMatchImage Type = ResourceManager Description = "Select MATCH image" Operations = ( ResetDiscard ) EndObject # Apply overlap ratio limits to all images (will implicity exclude MATCH # image because of NULL in OverlapRatio* columns in the input CSV) Object = Strategy Name = ConstrainOverlapRatios Type = Limit Description = "Apply image constraints to all images" # This set of equations will result in 1 if test are good, meaning the # constraints all pass. If any test fails, InRegion = 0 # (false) and the Resource (overlapping image) is disabled. Group = Equations InRegion = 1 # Needs at least a 3.50% overlap ratio (doesn't apply to MATCH as # these values are not defined for it and it is implicitly discarded # at this point) InRegion = "InRegion * (OverlapRatioSource > 0.0350)" InRegion = "InRegion * (OverlapRatioMatch > 0.0350)" EndGroup EndObject # Delete all non-passing resources which could also be the MATCH file Object = Strategy Name = RemoveBadOverlaps Type = ResourceManager Description = "Remove all bad overlaps for FROMLIST and the MATCH file" Operations = ( DeleteDiscard ) EndObject # Now just have FROMLIST Resources - create input conditions for matcher Object = Strategy Name = MultiImageMatcher Type = RunCommand Description = "Create findfeatures command and execute..." SkipCommandsIfNoData = True OnPreCommandErrorContinue = False OnPostCommandErrorContinue = False CommandArgs = ( ImageDir, OutputDir, Region, SourceProductId, YearDoy, File, MatchedProductId, multi_algorithms ) # Setup commands for the findfeatures run. Got to be at least one FROM. # Creates output directory {OutputDir}/{Region}_Networks. Group = PreCommands Mkdir = "mkdir -p %2/%3_Networks" DeleteFrom = "/bin/rm -f %2/%3_Networks/%4_fromlist.lis" DeleteCmd0 = "/bin/rm -f %2/%3_Networks/%4_matcher.cmd" DeleteCmd1 = "/bin/rm -f %2/%3_Networks/%4_notmatched.lis" Touch = "touch %2/%3_Networks/%4_fromlist.lis" EndGroup # Append the current FROMLIST file to matcher input list Group = Commands Append = "echo "%1/%5/%6" >> %2/%3_Networks/%7_fromlist.lis" EndGroup # Set up the multi-image matcher run and execute it Group = PostCommands Start = "echo "Making multi-matcher for %4..."" Program = "echo "findfeatures algospecfile=%8 match=%1/%5/%6 fromlist=%2/%3_Networks/%4_fromlist.lis fastgeom=true geomtype=camera epitolerance=1.0 ratio=0.90 hmgtolerance=1.0 networkid=%3 pointid='%4_?????' onet=%2/%3_Networks/%4.net tolist=%2/%3_Networks/%4_cubes.lis tonotmatched=%2/%3_Networks/%4_notmatched.lis description='Create image-image control network' debug=false debuglog=%2/%3_Networks/%4.log" > %2/%3_Networks/%4_matcher.cmd" Report = "echo "Running multi-matcher for %4..."" Matchem = "source %2/%3_Networks/%4_matcher.cmd" Copyem = "cp -p %2/%3_Networks/%4.net %2/%3_Networks/%4_combined.net" EndGroup EndObject EndObject EndObject # Make some keywords for processing singleton networks Object = Strategy Name = KeywordMaker Type = Calculator Description = "Make some keys..." InitializersArgs = ( ImageDir, OutputDir, Region, SourceProductId, YearDoy, File, MatchedProductId, multi_algorithms ) Group = Initializers SourceId = "%4" Query = "%1/%5/%6" EndGroup # Specify search ranges for cnetcombinept Group = Equations GroundTol = "PixelResolution * 3.0" SearchRadius = "PixelResolution * 10.0" EndGroup EndObject # With the MATCH file in the global variable pool here again, we will # run through the list of failed ones individually and merge the networks # when complete Object = Strategy Name = RunMatcher Type = AssetSidebar Asset = FailedList Operation = Create CreateSource = Copy Object = Isisminer Name = SingleMatcherLogic # First clear all Resources and prepare to load remaining images to match Object = Strategy Name = ClearAll Type = ResourceManager Description = "Clear out all resources..." Operations = ( ResetDiscard, ToggleDiscard, DeleteDiscard ) EndObject # Read failed match file and process failed list with adjustments to # algorithm parameters Object = Strategy Name = ReadFailedImages Type = CsvReader Description = "Read bad list for processing" CsvFileArgs = ( ImageDir, OutputDir, Region, SourceProductId, YearDoy, File, MatchedProductId, single_algorithms, SourceId ) CsvFile = "%2/%3_Networks/%4_notmatched.lis" # Its just a list of image file names HasHeader = false ColumnBaseName = "Column" RowBaseName = "S" EndObject # Now just have FROMLIST Resources - create input conditions for matcher. # This set of commands runs the findfeature algorithms for pair-wise # singleton networks. # # This strategy will take the truth image and run single pair matching # with additional parameters. A Scharr filter is run on both images to # enhance edge detection. Successful pair-wise matching results in output # files of the form {OutputDir}/{Region}_Networks/{SourceProductId}_S#.net # where # corresponds to the row number of images listed in the notmatch # file in the previous multi-match mode. Object = Strategy Name = SingleImageMatcher Type = RunCommand Description = "Run on each failed image with special parameters" SkipCommandsIfNoData = True OnPreCommandErrorContinue = False OnPostCommandErrorContinue = False # Parameters substitutions for all commands CommandArgs = ( ImageDir, OutputDir, Region, SourceProductId, YearDoy, File, MatchedProductId, single_algorithms, Column0, Identity, SourceId , Query, SearchRadius, GroundTol ) # Setup commands for the findfeatures run. Got to be at least one FROM. Group = PreCommands DeleteCmd0 = "/bin/rm -f %2/%3_Networks/%4_%10_matcher.cmd" DeleteCmd1 = "/bin/rm -f %2/%3_Networks/%4_singles_notmatched.lis" Touch = "touch %2/%3_Networks/%4_singlenets.lis" EndGroup # Match single pairs (singletons) of images directly Group = Commands Report = "echo "Making single matcher on %9..."" Program = "echo "findfeatures algospecfile=%8 match=%12 from=%9 fastgeom=true geomtype=crop filter=scharr epitolerance=1.0 ratio=0.99 hmgtolerance=1.0 networkid=%3_%10 pointid='%4_%10_?????' onet=%2/%3_Networks/%4_%10.net tonotmatched=%2/%3_Networks/%4_singles_notmatched.lis description='Create image-image control network' debug=false debuglog=%2/%3_Networks/%4_%10.log" > %2/%3_Networks/%4_%10_matcher.cmd" # Execute the single matcher command Report = "echo "Running single matcher for %4..."" Matchem = "source %2/%3_Networks/%4_%10_matcher.cmd" # Append to network cubes list AddCube = "echo "%9" >> %2/%3_Networks/%4_cubes.lis" AddNet = "echo "%2/%3_Networks/%4_%10.net" >> %2/%3_Networks/%4_singlenets.lis" # Files may not be needed/used # Cleanup = "/bin/rm -f %2/%3_Networks/%4_%10_matcher.cmd # %2/%3_Networks/%4_%10.log" EndGroup # Combine singleton networks into one net with 3 measure mininum. Then # combine singletons with initial multi-match network w/2 measure # minimum Group = PostCommands Combine1 = "cnetcombinept cnetlist=%2/%3_Networks/%4_singlenets.lis imagetol=1.0 distance=%13 minmeasures=3 onet=%2/%3_Networks/%4_singletons.net" # Combine comprehensive singleton network to multi-matched network Combine2 = "cnetcombinept cnetbase=%2/%3_Networks/%4.net cnetfrom=%2/%3_Networks/%4_singletons.net imagetol=1.0 distance=%13 groundtol=%14 onet=%2/%3_Networks/%4_combined.net " EndGroup EndObject EndObject EndObject EndObject End