From eaf2e4deb852b4761ba13a3452bb51aa9495a583 Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Wed, 23 Jan 2019 10:54:54 -0500 Subject: [PATCH 1/4] Add contour angle to contours report Angle is from minAreaRect, and is in the range [-90, 0) Rework contours operations to be a bit more functional Make filter contours a bit more optimized --- .../operations/composite/ContoursReport.java | 92 ++++++++++--------- .../composite/FilterContoursOperation.java | 29 ++++-- .../java/edu/wpi/grip/core/util/LazyInit.java | 44 +++++++++ .../edu/wpi/grip/core/util/PointerStream.java | 26 ++++++ .../edu/wpi/grip/core/util/LazyInitTest.java | 45 +++++++++ .../wpi/grip/core/util/PointerStreamTest.java | 38 ++++++++ 6 files changed, 219 insertions(+), 55 deletions(-) create mode 100644 core/src/main/java/edu/wpi/grip/core/util/LazyInit.java create mode 100644 core/src/main/java/edu/wpi/grip/core/util/PointerStream.java create mode 100644 core/src/test/java/edu/wpi/grip/core/util/LazyInitTest.java create mode 100644 core/src/test/java/edu/wpi/grip/core/util/PointerStreamTest.java diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/ContoursReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/ContoursReport.java index 555ddc1058..3a60b7419e 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/ContoursReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/ContoursReport.java @@ -5,17 +5,21 @@ import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; import edu.wpi.grip.core.sockets.Socket; +import edu.wpi.grip.core.util.LazyInit; +import edu.wpi.grip.core.util.PointerStream; import com.google.auto.value.AutoValue; +import org.bytedeco.javacpp.opencv_core.RotatedRect; +import org.bytedeco.javacpp.opencv_imgproc; + import java.util.ArrayList; import java.util.List; -import java.util.Optional; +import java.util.stream.Stream; import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_core.MatVector; import static org.bytedeco.javacpp.opencv_core.Rect; -import static org.bytedeco.javacpp.opencv_imgproc.boundingRect; import static org.bytedeco.javacpp.opencv_imgproc.contourArea; import static org.bytedeco.javacpp.opencv_imgproc.convexHull; @@ -31,7 +35,9 @@ public final class ContoursReport implements Publishable { private final int rows; private final int cols; private final MatVector contours; - private Optional boundingBoxes = Optional.empty(); + private final LazyInit boundingBoxes = new LazyInit<>(this::computeBoundingBoxes); + private final LazyInit rotatedBoundingBoxes = + new LazyInit<>(this::computeMinAreaBoundingBoxes); /** * Construct an empty report. This is used as a default value for {@link Socket}s containing @@ -70,9 +76,10 @@ public List getProcessedContours() { double[] width = getWidth(); double[] height = getHeights(); double[] solidity = getSolidity(); + double[] angles = getAngles(); for (int i = 0; i < contours.size(); i++) { processedContours.add(Contour.create(area[i], centerX[i], centerY[i], width[i], height[i], - solidity[i])); + solidity[i], angles[i])); } return processedContours; } @@ -82,66 +89,51 @@ public List getProcessedContours() { * boxes are used to compute several different properties, so it's probably not a good idea to * compute them over and over again. */ - private synchronized Rect[] computeBoundingBoxes() { - if (!boundingBoxes.isPresent()) { - Rect[] bb = new Rect[(int) contours.size()]; - for (int i = 0; i < contours.size(); i++) { - bb[i] = boundingRect(contours.get(i)); - } - - boundingBoxes = Optional.of(bb); - } + private Rect[] computeBoundingBoxes() { + return PointerStream.ofMatVector(contours) + .map(opencv_imgproc::boundingRect) + .toArray(Rect[]::new); + } - return boundingBoxes.get(); + private RotatedRect[] computeMinAreaBoundingBoxes() { + return PointerStream.ofMatVector(contours) + .map(opencv_imgproc::minAreaRect) + .toArray(RotatedRect[]::new); } @PublishValue(key = "area", weight = 0) public double[] getArea() { - final double[] areas = new double[(int) contours.size()]; - for (int i = 0; i < contours.size(); i++) { - areas[i] = contourArea(contours.get(i)); - } - return areas; + return PointerStream.ofMatVector(contours) + .mapToDouble(opencv_imgproc::contourArea) + .toArray(); } @PublishValue(key = "centerX", weight = 1) public double[] getCenterX() { - final double[] centers = new double[(int) contours.size()]; - final Rect[] boundingBoxes = computeBoundingBoxes(); - for (int i = 0; i < contours.size(); i++) { - centers[i] = boundingBoxes[i].x() + boundingBoxes[i].width() / 2; - } - return centers; + return Stream.of(boundingBoxes.get()) + .mapToDouble(r -> r.x() + r.width() / 2) + .toArray(); } @PublishValue(key = "centerY", weight = 2) public double[] getCenterY() { - final double[] centers = new double[(int) contours.size()]; - final Rect[] boundingBoxes = computeBoundingBoxes(); - for (int i = 0; i < contours.size(); i++) { - centers[i] = boundingBoxes[i].y() + boundingBoxes[i].height() / 2; - } - return centers; + return Stream.of(boundingBoxes.get()) + .mapToDouble(r -> r.y() + r.height() / 2) + .toArray(); } @PublishValue(key = "width", weight = 3) public synchronized double[] getWidth() { - final double[] widths = new double[(int) contours.size()]; - final Rect[] boundingBoxes = computeBoundingBoxes(); - for (int i = 0; i < contours.size(); i++) { - widths[i] = boundingBoxes[i].width(); - } - return widths; + return Stream.of(boundingBoxes.get()) + .mapToDouble(Rect::width) + .toArray(); } @PublishValue(key = "height", weight = 4) public synchronized double[] getHeights() { - final double[] heights = new double[(int) contours.size()]; - final Rect[] boundingBoxes = computeBoundingBoxes(); - for (int i = 0; i < contours.size(); i++) { - heights[i] = boundingBoxes[i].height(); - } - return heights; + return Stream.of(boundingBoxes.get()) + .mapToDouble(Rect::height) + .toArray(); } @PublishValue(key = "solidity", weight = 5) @@ -156,11 +148,19 @@ public synchronized double[] getSolidity() { return solidities; } + @PublishValue(key = "angle", weight = 6) + public synchronized double[] getAngles() { + return Stream.of(rotatedBoundingBoxes.get()) + .mapToDouble(RotatedRect::angle) + .toArray(); + } + @AutoValue public abstract static class Contour { public static Contour create(double area, double centerX, double centerY, double width, double - height, double solidity) { - return new AutoValue_ContoursReport_Contour(area, centerX, centerY, width, height, solidity); + height, double solidity, double angle) { + return new AutoValue_ContoursReport_Contour(area, centerX, centerY, width, height, solidity, + angle); } public abstract double area(); @@ -174,5 +174,7 @@ public static Contour create(double area, double centerX, double centerY, double public abstract double height(); public abstract double solidity(); + + public abstract double angle(); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FilterContoursOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FilterContoursOperation.java index 7afbf03ab7..42d565010e 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FilterContoursOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FilterContoursOperation.java @@ -15,9 +15,7 @@ import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_core.MatVector; -import static org.bytedeco.javacpp.opencv_core.Rect; import static org.bytedeco.javacpp.opencv_imgproc.arcLength; -import static org.bytedeco.javacpp.opencv_imgproc.boundingRect; import static org.bytedeco.javacpp.opencv_imgproc.contourArea; import static org.bytedeco.javacpp.opencv_imgproc.convexHull; @@ -74,6 +72,8 @@ public class FilterContoursOperation implements Operation { private final SocketHint maxRatioHint = SocketHints.Inputs.createNumberSpinnerSocketHint("Max Ratio", 1000, 0, Integer.MAX_VALUE); + private final SocketHint> angleHint = + SocketHints.Inputs.createNumberListRangeSocketHint("Angle", -90, 0); private final InputSocket contoursSocket; private final InputSocket minAreaSocket; @@ -87,6 +87,7 @@ public class FilterContoursOperation implements Operation { private final InputSocket maxVertexSocket; private final InputSocket minRatioSocket; private final InputSocket maxRatioSocket; + private final InputSocket> angleSocket; private final OutputSocket outputSocket; @@ -106,6 +107,7 @@ public FilterContoursOperation(InputSocket.Factory inputSocketFactory, OutputSoc this.maxVertexSocket = inputSocketFactory.create(maxVertexHint); this.minRatioSocket = inputSocketFactory.create(minRatioHint); this.maxRatioSocket = inputSocketFactory.create(maxRatioHint); + this.angleSocket = inputSocketFactory.create(angleHint); this.outputSocket = outputSocketFactory.create(contoursHint); } @@ -124,7 +126,8 @@ public List getInputSockets() { maxVertexSocket, minVertexSocket, minRatioSocket, - maxRatioSocket + maxRatioSocket, + angleSocket ); } @@ -139,6 +142,7 @@ public List getOutputSockets() { @SuppressWarnings("unchecked") public void perform() { final InputSocket inputSocket = contoursSocket; + final ContoursReport report = inputSocket.getValue().get(); final double minArea = minAreaSocket.getValue().get().doubleValue(); final double minPerimeter = minPerimeterSocket.getValue().get().doubleValue(); final double minWidth = minWidthSocket.getValue().get().doubleValue(); @@ -151,9 +155,10 @@ public void perform() { final double maxVertexCount = maxVertexSocket.getValue().get().doubleValue(); final double minRatio = minRatioSocket.getValue().get().doubleValue(); final double maxRatio = maxRatioSocket.getValue().get().doubleValue(); + final double minAngle = angleSocket.getValue().get().get(0).doubleValue(); + final double maxAngle = angleSocket.getValue().get().get(1).doubleValue(); - - final MatVector inputContours = inputSocket.getValue().get().getContours(); + final MatVector inputContours = report.getContours(); final MatVector outputContours = new MatVector(inputContours.size()); final Mat hull = new Mat(); @@ -164,15 +169,14 @@ public void perform() { for (int i = 0; i < inputContours.size(); i++) { final Mat contour = inputContours.get(i); - final Rect bb = boundingRect(contour); - if (bb.width() < minWidth || bb.width() > maxWidth) { + if (report.getWidth()[i] < minWidth || report.getWidth()[i] > maxWidth) { continue; } - if (bb.height() < minHeight || bb.height() > maxHeight) { + if (report.getHeights()[i] < minHeight || report.getHeights()[i] > maxHeight) { continue; } - final double area = contourArea(contour); + final double area = report.getArea()[i]; if (area < minArea) { continue; } @@ -191,11 +195,16 @@ public void perform() { continue; } - final double ratio = (double) bb.width() / (double) bb.height(); + final double ratio = report.getWidth()[i] / report.getHeights()[i]; if (ratio < minRatio || ratio > maxRatio) { continue; } + final double angle = report.getAngles()[i]; + if (angle < minAngle || angle > maxAngle) { + continue; + } + outputContours.put(filteredContourCount++, contour); } diff --git a/core/src/main/java/edu/wpi/grip/core/util/LazyInit.java b/core/src/main/java/edu/wpi/grip/core/util/LazyInit.java new file mode 100644 index 0000000000..1c60b0cbfb --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/util/LazyInit.java @@ -0,0 +1,44 @@ +package edu.wpi.grip.core.util; + +import java.util.Objects; +import java.util.function.Supplier; + +/** + * A holder for data that gets lazily initialized. + * + * @param the type of held data + */ +public class LazyInit { + + private T value = null; + private final Supplier factory; + + /** + * Creates a new lazily initialized data holder. + * + * @param factory the factory to use to create the held value + */ + public LazyInit(Supplier factory) { + this.factory = Objects.requireNonNull(factory, "factory"); + } + + /** + * Gets the value, initializing it if it has not yet been created. + * + * @return the held value + */ + public T get() { + if (value == null) { + value = factory.get(); + } + return value; + } + + /** + * Clears the held value. The next call to {@link #get()} will re-instantiate the held value. + */ + public void clear() { + value = null; + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/util/PointerStream.java b/core/src/main/java/edu/wpi/grip/core/util/PointerStream.java new file mode 100644 index 0000000000..1ccbfff25f --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/util/PointerStream.java @@ -0,0 +1,26 @@ +package edu.wpi.grip.core.util; + +import java.util.stream.LongStream; +import java.util.stream.Stream; + +import static org.bytedeco.javacpp.opencv_core.Mat; +import static org.bytedeco.javacpp.opencv_core.MatVector; + +/** + * Utility class for streaming native vector wrappers like {@code MatVector} + * ({@code std::vector}) with the Java {@link Stream} API. + */ +public final class PointerStream { + + /** + * Creates a stream of {@code Mat} objects in a {@code MatVector}. + * + * @param vector the vector of {@code Mats} to stream + * + * @return a new stream object for the contents of the vector + */ + public static Stream ofMatVector(MatVector vector) { + return LongStream.range(0, vector.size()) + .mapToObj(vector::get); + } +} diff --git a/core/src/test/java/edu/wpi/grip/core/util/LazyInitTest.java b/core/src/test/java/edu/wpi/grip/core/util/LazyInitTest.java new file mode 100644 index 0000000000..8ac8ea0f4e --- /dev/null +++ b/core/src/test/java/edu/wpi/grip/core/util/LazyInitTest.java @@ -0,0 +1,45 @@ +package edu.wpi.grip.core.util; + +import org.junit.Test; + +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; + +public class LazyInitTest { + + @Test + public void testFactoryIsOnlyCalledOnce() { + final String output = "foo"; + final int[] count = {0}; + final Supplier factory = () -> { + count[0]++; + return output; + }; + + LazyInit lazyInit = new LazyInit<>(factory); + lazyInit.get(); + assertEquals(1, count[0]); + + lazyInit.get(); + assertEquals("Calling get() more than once should only call the factory once", 1, count[0]); + } + + @Test + public void testClear() { + final String output = "foo"; + final int[] count = {0}; + final Supplier factory = () -> { + count[0]++; + return output; + }; + LazyInit lazyInit = new LazyInit<>(factory); + lazyInit.get(); + assertEquals(1, count[0]); + + lazyInit.clear(); + lazyInit.get(); + assertEquals(2, count[0]); + } + +} diff --git a/core/src/test/java/edu/wpi/grip/core/util/PointerStreamTest.java b/core/src/test/java/edu/wpi/grip/core/util/PointerStreamTest.java new file mode 100644 index 0000000000..77d4ceb6d3 --- /dev/null +++ b/core/src/test/java/edu/wpi/grip/core/util/PointerStreamTest.java @@ -0,0 +1,38 @@ +package edu.wpi.grip.core.util; + +import org.bytedeco.javacpp.opencv_core.MatVector; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class PointerStreamTest { + + private MatVector vector; + + @Before + public void setupVector() { + vector = new MatVector(); + } + + @After + public void freeVector() { + vector.deallocate(); + } + + @Test + public void testStreamEmptyMatVector() { + vector.resize(0); + long size = PointerStream.ofMatVector(vector).count(); + assertEquals("MatVector of size 0 should result in an empty stream", 0, size); + } + + @Test + public void testStreamMatVectorWithContents() { + final int size = 4; + vector.resize(size); + long actual = PointerStream.ofMatVector(vector).count(); + assertEquals("MatVector of size 4 should have 4 stream elements", size, actual); + } +} From 5efa91e64fcd03a3205b0aa987742f06cd4748fc Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Wed, 23 Jan 2019 16:53:48 -0500 Subject: [PATCH 2/4] Update comments --- .../grip/core/operations/composite/ContoursReport.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/ContoursReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/ContoursReport.java index 3a60b7419e..d09b2e2a12 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/ContoursReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/ContoursReport.java @@ -85,9 +85,7 @@ public List getProcessedContours() { } /** - * Compute the bounding boxes of all contours (if they haven't already been computed). Bounding - * boxes are used to compute several different properties, so it's probably not a good idea to - * compute them over and over again. + * Compute the bounding boxes of all contours. Called lazily and cached by {@link #boundingBoxes}. */ private Rect[] computeBoundingBoxes() { return PointerStream.ofMatVector(contours) @@ -95,6 +93,10 @@ private Rect[] computeBoundingBoxes() { .toArray(Rect[]::new); } + /** + * Computes the minimum-area bounding boxes of all contours. Called lazily and cached by + * {@link #rotatedBoundingBoxes}. + */ private RotatedRect[] computeMinAreaBoundingBoxes() { return PointerStream.ofMatVector(contours) .map(opencv_imgproc::minAreaRect) From 5bd5f804114b83a631180c9f8a121da5ee33c928 Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Thu, 18 Apr 2019 19:16:33 -0400 Subject: [PATCH 3/4] Add private constructor to PointerStream --- core/src/main/java/edu/wpi/grip/core/util/PointerStream.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/java/edu/wpi/grip/core/util/PointerStream.java b/core/src/main/java/edu/wpi/grip/core/util/PointerStream.java index 1ccbfff25f..81bfb43218 100644 --- a/core/src/main/java/edu/wpi/grip/core/util/PointerStream.java +++ b/core/src/main/java/edu/wpi/grip/core/util/PointerStream.java @@ -12,6 +12,10 @@ */ public final class PointerStream { + private PointerStream() { + throw new UnsupportedOperationException("This is a utility class!"); + } + /** * Creates a stream of {@code Mat} objects in a {@code MatVector}. * From ce8933bd1adba78ab80b211a904c0e73abd02dd6 Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Thu, 18 Apr 2019 19:41:00 -0400 Subject: [PATCH 4/4] Update codegen for FilterContours --- .../ui/codegeneration/cpp/operations/Filter_Contours.vm | 5 ++++- .../ui/codegeneration/java/operations/Filter_Contours.vm | 7 ++++++- .../ui/codegeneration/python/operations/Filter_Contours.vm | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/cpp/operations/Filter_Contours.vm b/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/cpp/operations/Filter_Contours.vm index 64b90b93f8..f65bec0e9d 100644 --- a/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/cpp/operations/Filter_Contours.vm +++ b/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/cpp/operations/Filter_Contours.vm @@ -15,9 +15,10 @@ This creates the C++ OpenCV Filter Contours function * @param maxVertexCount maximum vertex Count. * @param minRatio minimum ratio of width to height. * @param maxRatio maximum ratio of width to height. + * @param angle the minimum and maximum angle of a contour * @param output vector of filtered contours. */ - void $className::#func($step ["inputContours", "minArea", "minPerimeter", "minWidth", "maxWidth", "minHeight", "maxHeight", "solidity", "maxVertexCount", "minVertexCount", "minRatio", "maxRatio", "output"]) { + void $className::#func($step ["inputContours", "minArea", "minPerimeter", "minWidth", "maxWidth", "minHeight", "maxHeight", "solidity", "maxVertexCount", "minVertexCount", "minRatio", "maxRatio", "angle", "output"]) { std::vector hull; output.clear(); for (std::vector contour: inputContours) { @@ -33,6 +34,8 @@ This creates the C++ OpenCV Filter Contours function if (contour.size() < minVertexCount || contour.size() > maxVertexCount) continue; double ratio = (double) bb.width / (double) bb.height; if (ratio < minRatio || ratio > maxRatio) continue; + double contourAngle = cv::minAreaRect(contour).angle; + if (contourAngle < angle[0] || contourAngle > angle[1]) continue; output.push_back(contour); } } diff --git a/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/java/operations/Filter_Contours.vm b/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/java/operations/Filter_Contours.vm index acb6f79cbb..13b40da736 100644 --- a/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/java/operations/Filter_Contours.vm +++ b/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/java/operations/Filter_Contours.vm @@ -15,12 +15,13 @@ This creates the java OpenCV Filter Contours function * @param minVertexCount minimum vertex Count of the contours * @param maxVertexCount maximum vertex Count * @param minRatio minimum ratio of width to height + * @param angle the minimum and maximum angle of a contour * @param maxRatio maximum ratio of width to height */ private void $tMeth.name($step.name())(List inputContours, double minArea, double minPerimeter, double minWidth, double maxWidth, double minHeight, double maxHeight, double[] solidity, double maxVertexCount, double minVertexCount, double - minRatio, double maxRatio, List output) { + minRatio, double maxRatio, double[] angle, List output) { final MatOfInt hull = new MatOfInt(); output.clear(); //operation @@ -45,6 +46,10 @@ This creates the java OpenCV Filter Contours function if (contour.rows() < minVertexCount || contour.rows() > maxVertexCount) continue; final double ratio = bb.width / (double)bb.height; if (ratio < minRatio || ratio > maxRatio) continue; + final MatOfPoint2f copy = new MatOfPoint2f(contour); + final RotatedRect rr = Imgproc.minAreaRect(copy); + copy.release(); + if (rr.angle < angle[0] || rr.angle > angle[1]) continue; output.add(contour); } } \ No newline at end of file diff --git a/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/python/operations/Filter_Contours.vm b/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/python/operations/Filter_Contours.vm index b03ed777d6..e2592c4b7c 100644 --- a/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/python/operations/Filter_Contours.vm +++ b/ui/src/main/resources/edu/wpi/grip/ui/codegeneration/python/operations/Filter_Contours.vm @@ -1,7 +1,7 @@ @staticmethod def $tMeth.name($step.name())(input_contours, min_area, min_perimeter, min_width, max_width, min_height, max_height, solidity, max_vertex_count, min_vertex_count, - min_ratio, max_ratio): + min_ratio, max_ratio, angle): """Filters out contours that do not meet certain criteria. Args: input_contours: Contours as a list of numpy.ndarray. @@ -16,6 +16,7 @@ max_vertex_count: Maximum vertex Count. min_ratio: Minimum ratio of width to height. max_ratio: Maximum ratio of width to height. + angle: The minimum and maximum allowable angles of a contour. Returns: Contours as a list of numpy.ndarray. """ @@ -40,5 +41,8 @@ ratio = (float)(w) / h if (ratio < min_ratio or ratio > max_ratio): continue + contourAngle = cv2.minAreaRect(contour).angle + if (contourAngle < angle[0] or contourAngle > angle[1]): + continue output.append(contour) return output \ No newline at end of file