import net.sourceforge.jiu.data.RGBIntegerImage; import java.util.LinkedList; /** This class implements the texture synthesis algorithm described * by the paper "Image Quilting for Texture Synthesis and Transfer". * This only knows how to make a new texture image (ie it * does not implement hole filling or other uses of the algorithm). * * This class adds some optional extensions to improve the quality * of the output given more run time. */ public class ImageQuilter { private RGBIntegerImage input; private int patchsize; private int overlapsize; public static int DEFAULT_PATCH_SIZE = 36; public static int DEFAULT_OVERLAP_SIZE = 6; private boolean allowHorizontalPaths; private double pathCostWeight; /** This sets up the algorithm. * @param input This is the texture to sample from. * @param patchsize This is the width (pixels) of the square patches used. * @param overlapsize This is the width (pixels) of the overlap region. * @param allowHorizontalPaths When finding min paths, * can the path travel along a stage? * @param pathCostWeight The SSD for the overlap region and the * min SSD path cost have the same range. The total * cost is then pathCost*pathCostWeight * plus ssd*(1-pathCostWeight). */ public ImageQuilter(RGBIntegerImage input, int patchsize, int overlapsize, boolean allowHorizontalPaths, double pathCostWeight){ this.overlapsize = overlapsize; this.patchsize = patchsize; this.input = input; this.allowHorizontalPaths = allowHorizontalPaths; this.pathCostWeight = pathCostWeight; } /** This synthesizes a new texture image with the given dimensions. */ public RGBIntegerImage synthesize(int outwidth, int outheight) { if( outwidth < patchsize || outheight < patchsize ) { throw new IllegalArgumentException("Output size is too small"); } // check to see if the output size is acceptable and fix it if not int patchcols = Math.round((float)(outwidth-patchsize) / (patchsize-overlapsize)); int patchrows = Math.round((float)(outheight-patchsize) / (patchsize-overlapsize)); int okwidth = patchcols * (patchsize-overlapsize) + patchsize; int okheight = patchrows * (patchsize-overlapsize) + patchsize; patchcols++; patchrows++; if( okwidth != outwidth || okheight != outheight ) { System.out.println("Your output size requires partial" +" patches that are currently" +" not supported."); outwidth = okwidth; outheight = okheight; System.out.println("Using width = "+outwidth+" and height = " +outheight+" instead."); } // create the output image RGBIntegerImage output = null; output = (RGBIntegerImage) input.createCompatibleImage(outwidth, outheight); output.clear(0); // choose a patch at random to get started int x = (int) (Math.random() * (input.getWidth()-patchsize)); int y = (int) (Math.random() * (input.getHeight()-patchsize)); View inView = new View(input, x, y); Patch outPatch = new Patch(output, 0, 0, patchsize, patchsize); SynthAide.copy(inView, outPatch, 0, 0, patchsize, patchsize); // done already? if( !outPatch.nextColumn(overlapsize) ) return output; // loop over the rows of output patches int currow = 0; double dists[][] = new double[input.getHeight()-patchsize+1] [input.getWidth()-patchsize+1]; do { // loop over the patches in this row do { // get the distances for this neighborhood TwoDLoc bestloc = calcDists(dists, outPatch); double bestval = dists[bestloc.getRow()][bestloc.getCol()]; // pick one of the close matches double threshold = bestval * 1.1; LinkedList loclist = SynthAide.lessThanEqual(dists, threshold); int choice = (int) (Math.random() * loclist.size()); TwoDLoc loc = (TwoDLoc) loclist.get(choice); // copy in the patch //fillAndBlend(outPatch, loc); pathAndFill(outPatch, loc); } while( outPatch.nextColumn(overlapsize) ); currow++; System.out.println("done with row "+currow+" / "+patchrows); } while( outPatch.nextRow(overlapsize) ); return output; } /** This calculates the difference between the path costs of the left * overlap region and the top overlap region. */ private double getOverlapDistDifference(Patch outPatch, TwoDLoc loc) { Patch inPatch = new Patch(input, loc.getX(), loc.getY(), patchsize, patchsize); double left = 0.0, top = 0.0; double tolap[][] = getTopOverlapDists(outPatch, inPatch); double lolap[][] = getLeftOverlapDists(outPatch, inPatch); for(int r = 0; r < lolap.length; r++) { for(int c = 0; c < lolap[r].length; c++) { top += tolap[r][c]; left += lolap[r][c]; } } return (top > left ? top-left : left-top); } /** This calculates the distance (SSD) between the overlap part of outPatch * and the corresponding parts of the possible input patches. * If the pathCostWeight extension has been activated, this will also * calculate the path cost and weight the distance based on that cost. * This returns the array index of the smallest distance found. * @param dists This will be filled in. The return value in dists[y][x] * will be the SSD between an input patch with corner * (x,y) and the given output patch. */ private TwoDLoc calcDists(double[][] dists, Patch outPatch) { double best = Double.MAX_VALUE; TwoDLoc bestloc = null; // loop over the possible input patch row locations Patch inPatch = new Patch(input, 0, 0, patchsize, patchsize); do { // loop over the possible input patch column locations do { double sum = 0.0; double leftoverlap[][] = null; double topoverlap[][] = null; int count = 0; // handle the left overlap part if( ! outPatch.isAtLeftEdge() ) { leftoverlap = getLeftOverlapDists(outPatch,inPatch); for(int r = 0; r < leftoverlap.length; r++) { for(int c = 0; c < leftoverlap[r].length; c++) { sum += leftoverlap[r][c]; } } count += leftoverlap.length * leftoverlap[0].length; } // handle the top overlap part if( ! outPatch.isAtTopEdge() ) { topoverlap = getTopOverlapDists(outPatch,inPatch); for(int r = 0; r < topoverlap.length; r++) { for(int c = 0; c < topoverlap[r].length; c++) { sum += topoverlap[r][c]; } } count += topoverlap.length * topoverlap[0].length; } // don't double count the upper left corner; if( leftoverlap != null && topoverlap != null ) { for(int x = 0; x < overlapsize; x++) { for(int y = 0; y < overlapsize; y++) { sum -= SynthAide.ssd(outPatch,inPatch,x,y) / 3.0; } } count -= overlapsize * overlapsize; } // make this an average SSD instead sum = sum / (count * 255 * 255); // do we weight the SSD with the min cost path cost? if( pathCostWeight > 0 ) { double cost = avgCostOfBestPath(leftoverlap,topoverlap); // update the sum appropriately cost = cost / (255 * 255); sum = sum * (1-pathCostWeight) + pathCostWeight * cost; } // save the total and compare to the best yet int y = inPatch.getCornerY(); int x = inPatch.getCornerX(); dists[y][x] = sum; if( sum < best ) { best = sum; bestloc = new TwoDLoc(y,x); } } while( inPatch.rightOnePixel() ); } while( inPatch.nextPixelRow() ); return bestloc; } /** This returns the cost of the path through the given overlap region * divided by the length of the path. */ private double avgCostOfBestPath(double[][] leftoverlap, double[][] topoverlap) { double cost; int rowcnt; if( leftoverlap == null ) { MinPathFinder tpath = new MinPathFinder(topoverlap, allowHorizontalPaths); TwoDLoc loc = tpath.bestSourceLoc(); cost = tpath.costOf(loc.getRow(), loc.getCol()); rowcnt = patchsize; } else if( topoverlap == null ) { MinPathFinder lpath = new MinPathFinder(leftoverlap, allowHorizontalPaths); TwoDLoc loc = lpath.bestSourceLoc(); cost = lpath.costOf(loc.getRow(), loc.getCol()); rowcnt = patchsize; } else { MinPathFinder lpath = new MinPathFinder(leftoverlap, allowHorizontalPaths); MinPathFinder tpath = new MinPathFinder(topoverlap, allowHorizontalPaths); TwoDLoc lloc = new TwoDLoc(0,0); TwoDLoc tloc = new TwoDLoc(0,0); choosePathIntersection(lpath, tpath, lloc, tloc); // what is the total cost of the two paths? // this ignores the fact that the two have a pt in common cost = lpath.costOf(lloc.getRow(),lloc.getCol()); cost += tpath.costOf(tloc.getRow(),tloc.getCol()); // what is the combined length of the two paths rowcnt = 2*patchsize-2-lloc.getRow()-tloc.getRow(); rowcnt = 2 * patchsize - rowcnt; } return cost / rowcnt; } /** This creates an array the size of the horizontal overlap region * and fills that array with the SSDs between the patches in that * region. The array returned is upside down, such that array[0][0] * is the lower left corner of the overlap region. * (it's that way to be convenient input to MinPathFinder) */ private double[][] getLeftOverlapDists(Patch outPatch, Patch inPatch) { int rowcnt = outPatch.getHeight(); double dists[][] = new double[rowcnt][overlapsize]; int arrayr = rowcnt-1; for(int r = 0; r < rowcnt; r++) { for(int c = 0; c < overlapsize; c++) { dists[arrayr][c] = SynthAide.ssd(outPatch, inPatch, c, r) / 3; } arrayr--; } return dists; } /** This creates an array the size of the horizontal overlap region * and fills that array with the SSDs between the patches in that * region. The array is set up to be vertical with one array row * per column of the overlap region and array[0][0] being the * upper right corner of the overlap region. */ private double[][] getTopOverlapDists(Patch outPatch, Patch inPatch) { // so arrayr = patchwidth-1-patchx and arrayc = patchy int rowcnt = outPatch.getWidth(); double dists[][] = new double[rowcnt][overlapsize]; for(int patchx = 0; patchx < rowcnt; patchx++) { int arrayr = rowcnt - patchx - 1; for(int patchy = 0; patchy < overlapsize; patchy++) { dists[arrayr][patchy] = SynthAide.ssd(outPatch, inPatch, patchx, patchy) / 3; } } return dists; } /** This copies a patch from the input image at location loc * into outPatch. The overlap regions will be blended. */ private void fillAndBlend(Patch outPatch, TwoDLoc loc) { Patch inPatch = new Patch(input, loc.getX(), loc.getY(), patchsize, patchsize); if( outPatch.isAtTopEdge() ) { // blend the overlap area on the left for(int r = 0; r < patchsize; r++) { for(int c = 0; c < overlapsize; c++) { double inpart = (double) c / overlapsize; SynthAide.blend(inPatch, outPatch, c, r, inpart); } } SynthAide.copy(inPatch, outPatch, overlapsize, 0, patchsize-overlapsize, overlapsize); } else if( outPatch.isAtLeftEdge() ) { // blend the overlap area on top for(int c = 0; c < patchsize; c++) { for(int r = 0; r < overlapsize; r++) { double inpart = (double) r / overlapsize; SynthAide.blend(inPatch, outPatch, c, r, inpart); } } SynthAide.copy(inPatch, outPatch, 0, overlapsize, overlapsize, patchsize-overlapsize); } else { // blend the overlap area on top for(int c = overlapsize; c < patchsize; c++) { for(int r = 0; r < overlapsize; r++) { double inpart = (double) r / overlapsize; SynthAide.blend(inPatch, outPatch, c, r, inpart); } } // blend the overlap area on the left for(int r = overlapsize; r < patchsize; r++) { for(int c = 0; c < overlapsize; c++) { double inpart = (double) c / overlapsize; SynthAide.blend(inPatch, outPatch, c, r, inpart); } } // blend the combined overlap for(int r = 0; r < overlapsize; r++) { for(int c = 0; c < overlapsize; c++) { double inpart = (double)c*r/(overlapsize*overlapsize); SynthAide.blend(inPatch, outPatch, c, r, inpart); } } } // copy in the remaining part int size = patchsize - overlapsize; SynthAide.copy(inPatch, outPatch, overlapsize, overlapsize, size,size); } /** Uses the min path boundary method to fill in outPatch from the * input image patch at loc. */ private void pathAndFill(Patch outPatch, TwoDLoc loc) { boolean allow = allowHorizontalPaths; Patch inPatch = new Patch(input, loc.getX(), loc.getY(), patchsize, patchsize); if( outPatch.isAtLeftEdge() ) { SynthAide.copy(inPatch, outPatch, 0, 0, overlapsize, patchsize); double[][] topOverlap = getTopOverlapDists(outPatch, inPatch); MinPathFinder topFinder = new MinPathFinder(topOverlap, allow); TwoDLoc source = topFinder.bestSourceLoc(); followTopOverlapPath(outPatch, inPatch, topFinder, source); } else if( outPatch.isAtTopEdge() ) { SynthAide.copy(inPatch, outPatch, 0, 0, patchsize, overlapsize); double[][] leftOverlap = getLeftOverlapDists(outPatch, inPatch); MinPathFinder leftFinder = new MinPathFinder(leftOverlap, allow); TwoDLoc source = leftFinder.bestSourceLoc(); followLeftOverlapPath(outPatch, inPatch, leftFinder, source); } else { double[][] topOverlap = getTopOverlapDists(outPatch, inPatch); double[][] leftOverlap = getLeftOverlapDists(outPatch, inPatch); MinPathFinder topFinder = new MinPathFinder(topOverlap, allow); MinPathFinder leftFinder = new MinPathFinder(leftOverlap, allow); TwoDLoc leftloc = new TwoDLoc(0,0); TwoDLoc toploc = new TwoDLoc(0,0); // find the best combined source choosePathIntersection(leftFinder, topFinder, leftloc, toploc); // fill in the corner // first figure out where to take each pixel from boolean where[][] = new boolean[overlapsize][overlapsize]; // figure out where the left overlap says to take each pixel from while( leftloc.getRow() < overlapsize ) { int r = leftloc.getRow(); for(int c = leftloc.getCol(); c < overlapsize; c++) { where[r][c] = true; } leftloc = leftFinder.follow(leftloc); } // figure out where the top overlap agrees with the left overlap while( toploc.getRow() < overlapsize ) { int r = toploc.getRow(); for(int c = 0; c < overlapsize; c++) { where[c][r] = where[c][r] && c >= toploc.getCol(); } toploc = topFinder.follow(toploc); } // fill in the corner for real now for(int r = 0; r < overlapsize; r++) { for(int c = 0; c < overlapsize; c++) { if( where[r][c] ) { outPatch.putSample( c, r, inPatch.getSample(c,r) ); } } } // handle the rest of the overlap regions followLeftOverlapPath(outPatch, inPatch, leftFinder, leftloc); followTopOverlapPath(outPatch, inPatch, topFinder, toploc); } // fill in the non-overlap area int size = patchsize - overlapsize; SynthAide.copy(inPatch, outPatch, overlapsize, overlapsize, size,size); } /** Fills in the left overlap area of toPatch using values from * fromPatch while following the path from source in finder. */ private void followLeftOverlapPath(Patch toPatch, Patch fromPatch, MinPathFinder finder, TwoDLoc source) { // loop until we reach the destination while( source != null ) { int y = patchsize - source.getRow() - 1; int x = source.getCol(); // values to the right of x are filled in from fromPatch for(x++; x < overlapsize; x++) { toPatch.putSample(x, y, fromPatch.getSample(x, y)); } // values at that low point are averaged x = source.getCol(); SynthAide.blend(fromPatch, toPatch, x, y, 0.5); //int red[] = {255,0,0}; // DEBUG //toPatch.putSample(x, y, red); // DEBUG // values to the left are untouched // continue to the next row // we should probably check for this ahead of time and blend // (instead of replace or ignore) all pixels along the path int oldrow = source.getRow(); do{ source = finder.follow(source); } while( source != null && source.getRow() == oldrow ); } } /** Fills in the top overlap area of toPatch using values from * fromPatch while following the path from source in finder. */ private void followTopOverlapPath(Patch toPatch, Patch fromPatch, MinPathFinder finder, TwoDLoc source) { // loop until we reach the destination while( source != null ) { int x = patchsize - source.getRow() - 1; int y = source.getCol(); // values below y are filled in from fromPatch for(y++; y < overlapsize; y++) { toPatch.putSample(x, y, fromPatch.getSample(x, y)); } // values at that low point are averaged y = source.getCol(); SynthAide.blend(fromPatch, toPatch, x, y, 0.5); //int red[] = {255,0,0}; // DEBUG //toPatch.putSample(x, y, red); // DEBUG // values above are untouched // continue to the next row // we should probably check for this ahead of time and blend // (instead of replace or ignore) all pixels along the path int oldrow = source.getRow(); do{ source = finder.follow(source); } while( source != null && source.getRow() == oldrow ); } } /** This finds the intersection of the two given paths. * The intersection point (in each path's coordinates) is put * into the leftloc and toploc params. */ private void choosePathIntersection(MinPathFinder leftpath, MinPathFinder toppath, TwoDLoc leftloc, TwoDLoc toploc) { // find the best combined source leftloc.set(patchsize-1,0); // upper left corner toploc.set(patchsize-1,0); // of the image double bestcost = leftpath.costOf(patchsize-1,0) + toppath.costOf(patchsize-1,0); for(int y = 0; y < overlapsize; y++) { for(int x = 0; x < overlapsize; x++) { double cost = leftpath.costOf(patchsize-1-y,x) +toppath.costOf(patchsize-1-x,y); if( bestcost > cost ) { leftloc.set(patchsize-1-y,x); toploc.set(patchsize-1-x,y); bestcost = cost; } } } } }