Use cscore for PublishVideoOperation #866
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,149 +7,141 @@ | |
| import edu.wpi.grip.core.sockets.SocketHints; | ||
| import edu.wpi.grip.core.util.Icon; | ||
|
|
||
| import com.google.common.annotations.VisibleForTesting; | ||
| import com.google.common.collect.ImmutableList; | ||
|
|
||
| import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; | ||
| import edu.wpi.cscore.CameraServerJNI; | ||
| import edu.wpi.cscore.CvSource; | ||
| import edu.wpi.cscore.MjpegServer; | ||
| import edu.wpi.cscore.VideoMode; | ||
| import edu.wpi.first.wpilibj.networktables.NetworkTable; | ||
| import edu.wpi.first.wpilibj.tables.ITable; | ||
|
|
||
| import org.bytedeco.javacpp.opencv_core; | ||
| import org.opencv.core.Mat; | ||
|
|
||
| import java.net.Inet4Address; | ||
| import java.net.NetworkInterface; | ||
| import java.net.SocketException; | ||
| import java.util.Collection; | ||
| import java.util.Collections; | ||
| import java.util.Deque; | ||
| import java.util.LinkedList; | ||
| import java.util.List; | ||
| import java.util.logging.Level; | ||
| import java.util.logging.Logger; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
|
|
||
| /** | ||
| * Publish an M-JPEG stream with the protocol used by SmartDashboard and the FRC Dashboard. This | ||
| * allows FRC teams to view video streams on their dashboard during competition even when GRIP has | ||
| * exclusive access to the camera. Uses cscore to host the image streaming server. | ||
| */ | ||
| public class PublishVideoOperation implements Operation { | ||
|
|
||
| private static final Logger logger = Logger.getLogger(PublishVideoOperation.class.getName()); | ||
|
|
||
| /** | ||
| * Flags whether or not cscore was loaded. If it could not be loaded, the MJPEG streaming server | ||
| * can't be started, preventing this operation from running. | ||
| */ | ||
| private static final boolean cscoreLoaded; | ||
|
|
||
| private static final List<NetworkInterface> networkInterfaces; | ||
|
|
||
| static { | ||
| boolean loaded; | ||
| try { | ||
| // Loading the CameraServerJNI class will load the appropriate platform-specific OpenCV JNI | ||
| CameraServerJNI.getHostname(); | ||
| loaded = true; | ||
| } catch (Throwable e) { | ||
|
There was a problem hiding this comment. Can native bindings ever throw an There was a problem hiding this comment. It's most likely to be There was a problem hiding this comment. Unless we explicitly throw interrupted from the JNI level, nothing in JNI otherwise would throw it. And since its a dumb idea to interrupt a thread anyway, we would never throw it internally. Pretty much all that shows up is UnsatisfiedLinkError, or one of the few runtime exceptions we throw. Note this should be using forceLoad(), not getHostname() (I don't think we had forceLoad() then), but I will my sure my version does that. |
||
| logger.log(Level.SEVERE, "CameraServerJNI load failed!", e); | ||
| loaded = false; | ||
| } | ||
| cscoreLoaded = loaded; | ||
|
|
||
| List<NetworkInterface> interfaces; | ||
| try { | ||
| interfaces = Collections.list(NetworkInterface.getNetworkInterfaces()); | ||
| } catch (SocketException e) { | ||
| logger.log(Level.SEVERE, "Could not get the local network interfaces", e); | ||
| interfaces = Collections.emptyList(); | ||
| } | ||
| networkInterfaces = interfaces; | ||
| } | ||
|
There was a problem hiding this comment. Does this need to happen statically? Can't this be done when the operation is constructed? There was a problem hiding this comment. Then the cscore loading check would happen every time a new operation is created instead of only once, when the class loads. I'd prefer to just have a flag set so the operation can be created normally and just throw an exception in |
||
|
|
||
| public static final OperationDescription DESCRIPTION = | ||
| OperationDescription.builder() | ||
| .name("Publish Video") | ||
| .summary("Publish an M_JPEG stream to the dashboard.") | ||
| .category(OperationDescription.Category.NETWORK) | ||
| .icon(Icon.iconStream("publish-video")) | ||
| .build(); | ||
| private static final int INITIAL_PORT = 1180; | ||
| private static final int MAX_STEP_COUNT = 10; // limit ports to 1180-1189 | ||
|
|
||
|
|
||
| @SuppressWarnings("PMD.AssignmentToNonFinalStatic") | ||
| private static int totalStepCount; | ||
| @SuppressWarnings("PMD.AssignmentToNonFinalStatic") | ||
| private static int numSteps; | ||
| private static final Deque<Integer> availablePorts = | ||
| Stream.iterate(INITIAL_PORT, i -> i + 1) | ||
| .limit(MAX_STEP_COUNT) | ||
| .collect(Collectors.toCollection(LinkedList::new)); | ||
|
|
||
| private final InputSocket<opencv_core.Mat> inputSocket; | ||
| private final InputSocket<Number> qualitySocket; | ||
| private final MjpegServer server; | ||
| private final CvSource serverSource; | ||
|
|
||
| // Write to the /CameraPublisher table so the MJPEG streams are discoverable by other | ||
| // applications connected to the same NetworkTable server (eg Shuffleboard) | ||
| private final ITable cameraPublisherTable = NetworkTable.getTable("/CameraPublisher"); // NOPMD | ||
| private final ITable ourTable; | ||
| private Mat publishMat = null; | ||
| private long lastFrame = -1; | ||
|
|
||
| @SuppressWarnings("JavadocMethod") | ||
| @SuppressFBWarnings(value = "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD", | ||
| justification = "Do not need to synchronize inside of a constructor") | ||
| public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { | ||
| if (numSteps >= MAX_STEP_COUNT) { | ||
| throw new IllegalStateException( | ||
| "Only " + MAX_STEP_COUNT + " instances of PublishVideoOperation may exist"); | ||
| } | ||
| this.inputSocket = inputSocketFactory.create(SocketHints.Inputs.createMatSocketHint("Image", | ||
| false)); | ||
| this.qualitySocket = inputSocketFactory.create(SocketHints.Inputs | ||
| .createNumberSliderSocketHint("Quality", 80, 0, 100)); | ||
|
|
||
| if (cscoreLoaded) { | ||
| int ourPort = availablePorts.removeFirst(); | ||
|
|
||
| server = new MjpegServer("GRIP video publishing server " + totalStepCount, ourPort); | ||
| serverSource = new CvSource("GRIP CvSource " + totalStepCount, | ||
| VideoMode.PixelFormat.kMJPEG, 0, 0, 0); | ||
| server.setSource(serverSource); | ||
|
|
||
| ourTable = cameraPublisherTable.getSubTable("GRIP-" + totalStepCount); | ||
| try { | ||
| List<NetworkInterface> networkInterfaces = | ||
| Collections.list(NetworkInterface.getNetworkInterfaces()); | ||
| ourTable.putStringArray("streams", generateStreams(networkInterfaces, ourPort)); | ||
| } catch (SocketException e) { | ||
| logger.log(Level.WARNING, "Could not enumerate the local network interfaces", e); | ||
| ourTable.putStringArray("streams", new String[0]); | ||
|
There was a problem hiding this comment. Should we log something here? Or maybe throw exceptions in There was a problem hiding this comment. Maybe? The documentation isn't very clear on what circumstances will cause this exception to be thrown. It might never be thrown for the local host since it appears to only happen if the host is unreachable. There was a problem hiding this comment. Interesting. @PeterJohnson do you have any thoughts on what might cause this on the local system or if this is a non-issue? There was a problem hiding this comment. It sounds like how the Java getLocalHost function works is it gets the local hostname, then resolves it into an address. So it's definitely possible to fail: https://stackoverflow.com/questions/1881546/inetaddress-getlocalhost-throws-unknownhostexception |
||
| } | ||
| } else { | ||
| server = null; | ||
| serverSource = null; | ||
| ourTable = null; | ||
| } | ||
|
|
||
| numSteps++; | ||
| totalStepCount++; | ||
| } | ||
|
|
||
| @Override | ||
|
|
@@ -167,25 +159,87 @@ public List<OutputSocket> getOutputSockets() { | |
|
|
||
| @Override | ||
| public void perform() { | ||
| final long now = System.nanoTime(); // NOPMD | ||
|
|
||
| if (!cscoreLoaded) { | ||
| throw new IllegalStateException( | ||
| "cscore could not be loaded. The image streaming server cannot be started."); | ||
|
|
||
| } | ||
|
|
||
| opencv_core.Mat input = inputSocket.getValue().get(); | ||
| if (input.empty() || input.isNull()) { | ||
| throw new IllegalArgumentException("Input image must not be empty"); | ||
| } | ||
|
|
||
| // "copy" the input data to an OpenCV mat for cscore to use | ||
| // This basically just wraps the mat pointer in a different wrapper object | ||
| // No copies are performed, but it means we have to be careful about making sure we use the | ||
| // same version of JavaCV and OpenCV to minimize the risk of binary incompatibility. | ||
| // This copy only needs to happen once, since the operation input image is always the same | ||
| // object that gets copied into. The data address will change, however, if the image is resized | ||
| // or changes type. | ||
| if (publishMat == null || publishMat.nativeObj != input.address()) { | ||
| publishMat = new Mat(input.address()); | ||
|
There was a problem hiding this comment. Can you keep the test you had for |
||
| } | ||
| // Make sure the output resolution is up to date. Might not be needed, depends on cscore updates | ||
| serverSource.setResolution(input.size().width(), input.size().height()); | ||
| serverSource.putFrame(publishMat); | ||
| if (lastFrame != -1) { | ||
| long dt = now - lastFrame; | ||
|
There was a problem hiding this comment. I've seen this actually be There was a problem hiding this comment. It'll actually be There was a problem hiding this comment.
There was a problem hiding this comment. In kotlin: There was a problem hiding this comment. It's technically infinite since the new frame comes at the same time as the previous one. But it doesn't happen here because it's rate limited by the pipeline and the update rate of the pipeline sources. I seriously doubt |
||
| serverSource.setFPS((int) (1e9 / dt)); | ||
| } | ||
| lastFrame = now; | ||
| } | ||
|
|
||
| @Override | ||
| public synchronized void cleanUp() { | ||
| numSteps--; | ||
| if (cscoreLoaded) { | ||
| availablePorts.addFirst(server.getPort()); | ||
| ourTable.getKeys().forEach(ourTable::delete); | ||
| serverSource.setConnected(false); | ||
| serverSource.free(); | ||
| server.free(); | ||
| if (publishMat != null) { | ||
| publishMat.release(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Generates an array of stream URLs that allow third-party applications to discover the | ||
| * appropriate URLs that can stream MJPEG. The URLs will all point to the same physical machine, | ||
| * but may use different network interfaces (eg WiFi and ethernet). | ||
| * | ||
| * @param networkInterfaces the local network interfaces | ||
| * @param serverPort the port the mjpeg streaming server is running on | ||
| * @return an array of URLs that can be used to connect to the MJPEG streaming server | ||
| */ | ||
| @VisibleForTesting | ||
| static String[] generateStreams(Collection<NetworkInterface> networkInterfaces, int serverPort) { | ||
| return networkInterfaces.stream() | ||
| .flatMap(i -> Collections.list(i.getInetAddresses()).stream()) | ||
| .filter(a -> a instanceof Inet4Address) // IPv6 isn't well supported, stick to IPv4 | ||
| .filter(a -> !a.isLoopbackAddress()) // loopback addresses only work for local processes | ||
| .distinct() | ||
| .flatMap(a -> Stream.of( | ||
| generateStreamUrl(a.getHostName(), serverPort), | ||
| generateStreamUrl(a.getHostAddress(), serverPort))) | ||
| .distinct() | ||
| .toArray(String[]::new); | ||
| } | ||
|
|
||
| /** | ||
| * Generates a URL that can be used to connect to an MJPEG stream provided by cscore. The host | ||
| * should be a non-loopback IPv4 address that is resolvable by applications running on non-local | ||
| * machines. | ||
| * | ||
| * @param host the server host | ||
| * @param port the port the server is running on | ||
| */ | ||
| @VisibleForTesting | ||
| static String generateStreamUrl(String host, int port) { | ||
| return String.format("mjpeg:http://%s:%d/?action=stream", host, port); | ||
| } | ||
|
|
||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Commented out because there's no
opencv-3.2.0-1.3with thelinux-frcclassifier and I'm worried about different memory models between binding versions (JavaCV with 3.0.0, OpenCV with 3.2.0) . This could get removed entirely if we remove the deploy functionality.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
With this commented out as it is is the deploy functionality essentially nerfed? If so, then we should remove the deployer for this season.