diff --git a/src/containers/stage.jsx b/src/containers/stage.jsx
index f3444af1a4a01ef3e577dbb36bb345264e42c79e..7e543617a30cade0d16ef6eaaac34e675c712c55 100644
--- a/src/containers/stage.jsx
+++ b/src/containers/stage.jsx
@@ -6,6 +6,7 @@ import VM from 'scratch-vm';
import {connect} from 'react-redux';
import {getEventXY} from '../lib/touch-utils';
+import {VideoProvider} from '../lib/camera';
import StageComponent from '../components/stage/stage.jsx';
@@ -57,6 +58,7 @@ class Stage extends React.Component {
this.renderer = new Renderer(this.canvas);
this.props.vm.attachRenderer(this.renderer);
this.props.vm.runtime.addListener('QUESTION', this.questionListener);
+ this.props.vm.setVideoProvider(new VideoProvider());
}
shouldComponentUpdate (nextProps, nextState) {
return this.props.width !== nextProps.width ||
diff --git a/src/lib/camera.js b/src/lib/camera.js
new file mode 100644
index 0000000000000000000000000000000000000000..339d59dc2ca3ac4696ce26738e0d16a4129ee133
--- /dev/null
+++ b/src/lib/camera.js
@@ -0,0 +1,377 @@
+import getUserMedia from 'get-user-media-promise';
+import log from './log.js';
+
+// Single Setup For All Video Streams used by the GUI
+// While VideoProvider uses a private _singleSetup
+// property to ensure that each instance of a VideoProvider
+// use the same setup, this ensures that all instances
+// of VideoProviders use a single stream. This way, closing a camera modal
+// does not affect the video on the stage, and a program running and disabling
+// video on the stage will not affect the camera modal's video.
+const requestStack = [];
+const requestVideoStream = videoDesc => {
+ let streamPromise;
+ if (requestStack.length === 0) {
+ streamPromise = getUserMedia({
+ audio: false,
+ video: videoDesc
+ });
+ requestStack.push(streamPromise);
+ } else if (requestStack.length > 0) {
+ streamPromise = requestStack[0];
+ requestStack.push(true);
+ }
+ return streamPromise;
+};
+
+const requestDisableCheck = () => {
+ requestStack.pop();
+ if (requestStack.length > 0) return false;
+ return true;
+};
+
+/**
+ * Video Manager for video extensions.
+ */
+class VideoProvider {
+ constructor () {
+ /**
+ * Default value for mirrored frames.
+ * @type boolean
+ */
+ this.mirror = true;
+
+ /**
+ * Cache frames for this many ms.
+ * @type number
+ */
+ this._frameCacheTimeout = 16;
+
+ /**
+ * DOM Video element
+ * @private
+ */
+ this._video = null;
+
+ /**
+ * Usermedia stream track
+ * @private
+ */
+ this._track = null;
+
+ /**
+ * Stores some canvas/frame data per resolution/mirror states
+ */
+ this._workspace = [];
+ }
+
+ static get FORMAT_IMAGE_DATA () {
+ return 'image-data';
+ }
+
+ static get FORMAT_CANVAS () {
+ return 'canvas';
+ }
+
+ /**
+ * Dimensions the video stream is analyzed at after its rendered to the
+ * sample canvas.
+ * @type {Array.<number>}
+ */
+ static get DIMENSIONS () {
+ return [480, 360];
+ }
+
+ /**
+ * Order preview drawable is inserted at in the renderer.
+ * @type {number}
+ */
+ static get ORDER () {
+ return 1;
+ }
+
+ /**
+ * Get the HTML video element containing the stream
+ */
+ get video () {
+ return this._video;
+ }
+
+ /**
+ * Request video be enabled. Sets up video, creates video skin and enables preview.
+ *
+ * ioDevices.video.requestVideo()
+ *
+ * @return {Promise.<Video>} resolves a promise to this IO device when video is ready.
+ */
+ enableVideo () {
+ this.enabled = true;
+ return this._setupVideo();
+ }
+
+ /**
+ * Disable video stream (turn video off)
+ */
+ disableVideo () {
+ this.enabled = false;
+ // If we have begun a setup process, call _teardown after it completes
+ if (this._singleSetup) {
+ this._singleSetup
+ .then(this._teardown.bind(this))
+ .catch(err => this.onError(err));
+ }
+ }
+
+ /**
+ * async part of disableVideo
+ * @private
+ */
+ _teardown () {
+ // we might be asked to re-enable before _teardown is called, just ignore it.
+ if (this.enabled === false) {
+ const disableTrack = requestDisableCheck();
+ this._singleSetup = null;
+ // by clearing refs to video and track, we should lose our hold over the camera
+ this._video = null;
+ if (this._track && disableTrack) {
+ this._track.stop();
+ }
+ this._track = null;
+ }
+ }
+
+ /**
+ * Return frame data from the video feed in a specified dimensions, format, and mirroring.
+ *
+ * @param {object} frameInfo A descriptor of the frame you would like to receive.
+ * @param {Array.<number>} frameInfo.dimensions [width, height] array of numbers. Defaults to [480,360]
+ * @param {boolean} frameInfo.mirror If you specificly want a mirror/non-mirror frame, defaults to the global
+ * mirror state (ioDevices.video.mirror)
+ * @param {string} frameInfo.format Requested video format, available formats are 'image-data' and 'canvas'.
+ * @param {number} frameInfo.cacheTimeout Will reuse previous image data if the time since capture is less than
+ * the cacheTimeout. Defaults to 16ms.
+ *
+ * @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors.
+ */
+ getFrame ({
+ dimensions = VideoProvider.DIMENSIONS,
+ mirror = this.mirror,
+ format = VideoProvider.FORMAT_IMAGE_DATA,
+ cacheTimeout = this._frameCacheTimeout
+ }) {
+ if (!this.videoReady) {
+ return null;
+ }
+ const [width, height] = dimensions;
+ const workspace = this._getWorkspace({dimensions, mirror: Boolean(mirror)});
+ const {videoWidth, videoHeight} = this._video;
+ const {canvas, context, lastUpdate, cacheData} = workspace;
+ const now = Date.now();
+
+ // if the canvas hasn't been updated...
+ if (lastUpdate + cacheTimeout < now) {
+
+ if (mirror) {
+ context.scale(-1, 1);
+ context.translate(width * -1, 0);
+ }
+
+ context.drawImage(this._video,
+ // source x, y, width, height
+ 0, 0, videoWidth, videoHeight,
+ // dest x, y, width, height
+ 0, 0, width, height
+ );
+
+ context.resetTransform();
+ workspace.lastUpdate = now;
+ }
+
+ // each data type has it's own data cache, but the canvas is the same
+ if (!cacheData[format]) {
+ cacheData[format] = {lastUpdate: 0};
+ }
+ const formatCache = cacheData[format];
+
+ if (formatCache.lastUpdate + cacheTimeout < now) {
+ if (format === VideoProvider.FORMAT_IMAGE_DATA) {
+ formatCache.lastData = context.getImageData(0, 0, width, height);
+ } else if (format === VideoProvider.FORMAT_CANVAS) {
+ // this will never change
+ formatCache.lastUpdate = Infinity;
+ formatCache.lastData = canvas;
+ } else {
+ log.error(`video io error - unimplemented format ${format}`);
+ // cache the null result forever, don't log about it again..
+ formatCache.lastUpdate = Infinity;
+ formatCache.lastData = null;
+ }
+
+ // rather than set to now, this data is as stale as it's canvas is
+ formatCache.lastUpdate = Math.max(workspace.lastUpdate, formatCache.lastUpdate);
+ }
+
+ return formatCache.lastData;
+ }
+
+ /**
+ * Method called when an error happens. Default implementation is just to log error.
+ *
+ * @abstract
+ * @param {Error} error An error object from getUserMedia or other source of error.
+ */
+ onError (error) {
+ log.error('Unhandled video io device error', error);
+ }
+
+ /**
+ * Create a video stream.
+ * Should probably be moved to -render or somewhere similar later
+ * @private
+ * @return {Promise} When video has been received, rejected if video is not received
+ */
+ _setupVideo () {
+ // We cache the result of this setup so that we can only ever have a single
+ // video/getUserMedia request happen at a time.
+ if (this._singleSetup) {
+ return this._singleSetup;
+ }
+
+ this._singleSetup = requestVideoStream({
+ width: {min: 480, ideal: 640},
+ height: {min: 360, ideal: 480}
+ })
+ .then(stream => {
+ this._video = document.createElement('video');
+ // Use the new srcObject API, falling back to createObjectURL
+ try {
+ this._video.srcObject = stream;
+ } catch (error) {
+ this._video.src = window.URL.createObjectURL(stream);
+ }
+ // Hint to the stream that it should load. A standard way to do this
+ // is add the video tag to the DOM. Since this extension wants to
+ // hide the video tag and instead render a sample of the stream into
+ // the webgl rendered Scratch canvas, another hint like this one is
+ // needed.
+ this._video.play(); // Needed for Safari/Firefox, Chrome auto-plays.
+ this._track = stream.getTracks()[0];
+ return this;
+ })
+ .catch(error => {
+ this._singleSetup = null;
+ this.onError(error);
+ });
+
+ return this._singleSetup;
+ }
+
+ get videoReady () {
+ if (!this.enabled) {
+ return false;
+ }
+ if (!this._video) {
+ return false;
+ }
+ if (!this._track) {
+ return false;
+ }
+ const {videoWidth, videoHeight} = this._video;
+ if (typeof videoWidth !== 'number' || typeof videoHeight !== 'number') {
+ return false;
+ }
+ if (videoWidth === 0 || videoHeight === 0) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * get an internal workspace for canvas/context/caches
+ * this uses some document stuff to create a canvas and what not, probably needs abstraction
+ * into the renderer layer?
+ * @private
+ * @return {object} A workspace for canvas/data storage. Internal format not documented intentionally
+ */
+ _getWorkspace ({dimensions, mirror}) {
+ let workspace = this._workspace.find(space => (
+ space.dimensions.join('-') === dimensions.join('-') &&
+ space.mirror === mirror
+ ));
+ if (!workspace) {
+ workspace = {
+ dimensions,
+ mirror,
+ canvas: document.createElement('canvas'),
+ lastUpdate: 0,
+ cacheData: {}
+ };
+ workspace.canvas.width = dimensions[0];
+ workspace.canvas.height = dimensions[1];
+ workspace.context = workspace.canvas.getContext('2d');
+ this._workspace.push(workspace);
+ }
+ return workspace;
+ }
+}
+
+/**
+ * Video Manager for Camera Modal
+ */
+class ModalVideoManager {
+ constructor (canvas) {
+ this._videoProvider = new VideoProvider();
+ /**
+ * Frame update interval
+ * @type number
+ */
+ this._frameTimeout = 16;
+
+ this._canvas = canvas;
+ // These values are double the stage dimensions so that the resulting
+ // image does not have to get sized down to accomodate double resolution
+ this._canvasWidth = 960; // Double Stage Width
+ this._canvasHeight = 720; // Double Stage Height
+
+ }
+
+ enableVideo () {
+ this._videoProvider.enableVideo().then(() => {
+
+ const ctx = this._canvas.getContext('2d');
+ ctx.scale(-1, 1);
+ ctx.translate(this._canvasWidth * -1, 0);
+
+ this._drawFrames();
+ });
+ }
+
+ _drawFrames () {
+ const video = this._videoProvider.video;
+ this._videoFeedInterval = setInterval(() =>
+ this._canvas.getContext('2d').drawImage(video,
+ // source x, y, width, height
+ 0, 0, video.videoWidth, video.videoHeight,
+ // dest x, y, width, height
+ 0, 0, this._canvasWidth, this._canvasHeight
+ ), this._frameTimeout);
+ }
+
+ takeSnapshot () {
+ clearInterval(this._videoFeedInterval);
+ return this._canvas.toDataURL('image/png');
+ }
+
+ clearSnapshot () {
+ this._drawFrames();
+ }
+
+ disableVideo () {
+ this._videoProvider.disableVideo();
+ }
+}
+
+export {
+ VideoProvider,
+ ModalVideoManager
+};