From 8c0d895cacbd728b43790cbbb10fb9c120e610cd Mon Sep 17 00:00:00 2001
From: Karishma Chadha <kchadha@scratch.mit.edu>
Date: Wed, 25 Apr 2018 23:09:00 -0400
Subject: [PATCH] Camera costume upload functionality. Needs UI and code
cleanup.
---
src/components/camera-modal/camera-modal.css | 118 ++++
src/components/camera-modal/camera-modal.jsx | 88 +++
src/components/camera-modal/icon--back.svg | Bin 0 -> 2848 bytes
src/containers/camera-modal.jsx | 89 +++
src/containers/costume-tab.jsx | 31 +-
src/containers/stage.jsx | 4 +-
src/lib/camera.js | 654 +++++++++++++++++++
src/lib/file-uploader.js | 3 +-
src/reducers/modals.js | 11 +
9 files changed, 994 insertions(+), 4 deletions(-)
create mode 100644 src/components/camera-modal/camera-modal.css
create mode 100644 src/components/camera-modal/camera-modal.jsx
create mode 100644 src/components/camera-modal/icon--back.svg
create mode 100644 src/containers/camera-modal.jsx
create mode 100644 src/lib/camera.js
diff --git a/src/components/camera-modal/camera-modal.css b/src/components/camera-modal/camera-modal.css
new file mode 100644
index 000000000..c86e7f0ca
--- /dev/null
+++ b/src/components/camera-modal/camera-modal.css
@@ -0,0 +1,118 @@
+@import "../../css/colors.css";
+@import "../../css/units.css";
+
+.modal-content {
+ width: 600px;
+}
+
+.body {
+ background: $ui-white;
+ padding: 1.5rem 2.25rem;
+}
+
+.visualization-container {
+ display: flex;
+ justify-content: space-around;
+}
+
+.camera-feed-container {
+ background: $ui-primary;
+ border: 1px solid $ui-black-transparent;
+ border-radius: 5px;
+ padding: 3px;
+
+}
+
+.camera-feed-container {
+ display: flex;
+ justify-content: space-around;
+ align-items: center;
+ width: 480px;
+ height: 360px;
+ position: relative;
+}
+
+.canvas {
+ width: 480px;
+ height: 380px;
+}
+
+.help-text {
+ margin: 10px auto 0;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+ color: rgb(167, 170, 181);
+ font-size: 0.95rem;
+ font-weight: 500;
+}
+
+.capture-text {
+ color: $motion-primary;
+}
+
+.main-button-row {
+ display: flex;
+ justify-content: space-around;
+ margin-top: 15px;
+}
+
+.main-button-row button {
+ padding: 0.5rem 0.75rem;
+ border-radius: 0.25rem;
+ background: transparent;
+ border: none;
+}
+
+.main-button-row button:disabled {
+ opacity: 0.25;
+}
+
+.main-button-row button:active, .main-button-row button:focus {
+ outline: none;
+}
+
+.button-row {
+ font-weight: bolder;
+ text-align: right;
+ display: flex;
+ justify-content: space-between;
+}
+
+.button-row button {
+ padding: 0.75rem 1rem;
+ border-radius: 0.25rem;
+ background: white;
+ border: 1px solid $ui-black-transparent;
+ font-weight: 600;
+ font-size: 0.85rem;
+ color: $motion-primary;
+}
+
+.button-row button.ok-button {
+ background: $motion-primary;
+ border: $motion-primary;
+ color: white;
+}
+
+.button-row button + button {
+ margin-left: 0.5rem;
+}
+
+.main-button {
+ text-align: center;
+}
+
+.capture-button {
+ overflow: visible;
+}
+
+.capture-button-circle {
+ fill: $motion-primary;
+ /* opacity: 0.25; */
+ stroke: $motion-tertiary;
+}
+
+.capture-button-circle-outline {
+ fill: $ui-black-transparent;
+ opacity: 0.25;
+ transition: 0.1s;
+}
diff --git a/src/components/camera-modal/camera-modal.jsx b/src/components/camera-modal/camera-modal.jsx
new file mode 100644
index 000000000..c39070c6a
--- /dev/null
+++ b/src/components/camera-modal/camera-modal.jsx
@@ -0,0 +1,88 @@
+import PropTypes from 'prop-types';
+import React from 'react';
+import Box from '../box/box.jsx';
+import Modal from '../modal/modal.jsx';
+import styles from './camera-modal.css';
+import backIcon from './icon--back.svg';
+
+const CameraModal = props => (
+ <Modal
+ className={styles.modalContent}
+ contentLabel={'Take a Picture'}
+ onRequestClose={props.onCancel}
+ >
+ <Box className={styles.body}>
+ <Box className={styles.visualizationContainer}>
+ <Box className={styles.cameraFeedContainer}>
+ <canvas
+ className={styles.canvas}
+ // height and (below) width of the actual image
+ height="720"
+ ref={props.canvasRef}
+ width="960"
+ />
+ </Box>
+ </Box>
+ {props.capture ?
+ <Box className={styles.buttonRow}>
+ <button
+ className={styles.cancelButton}
+ onClick={props.onBack}
+ >
+ <img
+ draggable={false}
+ src={backIcon}
+ /> Re-take
+ </button>
+ <button
+ className={styles.okButton}
+ onClick={props.onSubmit}
+ > Save
+ </button>
+ </Box> :
+ <Box className={styles.mainButtonRow}>
+ <button
+ className={styles.mainButton}
+ onClick={props.onCapture}
+ >
+ <svg
+ className={styles.captureButton}
+ height="52"
+ width="52"
+ >
+ <circle
+ className={styles.captureButtonCircle}
+ cx="26"
+ cy="26"
+ r="25"
+ />
+ <circle
+ className={styles.captureButtonCircleOutline}
+ cx="26"
+ cy="26"
+ r="27"
+ />
+ </svg>
+ <div className={styles.helpText}>
+ <span className={styles.captureText}>
+ {'Take Photo'}
+ </span>
+ </div>
+ </button>
+ </Box>
+ }
+ </Box>
+ </Modal>
+);
+
+CameraModal.propTypes = {
+ canvasRef: PropTypes.func.isRequired,
+ capture: PropTypes.instanceOf(ImageData),
+ onBack: PropTypes.func.isRequired,
+ onCancel: PropTypes.func.isRequired,
+ onCapture: PropTypes.func.isRequired,
+ onSubmit: PropTypes.func.isRequired
+ // videoRef: PropTypes.func.isRequired
+};
+
+export default CameraModal;
diff --git a/src/components/camera-modal/icon--back.svg b/src/components/camera-modal/icon--back.svg
new file mode 100644
index 0000000000000000000000000000000000000000..47d09bc7fad766b50da3ed1cffec190b1e94878c
GIT binary patch
literal 2848
zcmZ{m+fE}#6h+_XD>|K*AkpQzUq&_~#nI%2NTbZ;1037~7RHV24v@dkI@K6V#PkdH
zcCpVs`&_E$pPufv%|pLGY=+&UHA=Tlzq=aNo89%I{p;K3F1O8LT<zAY?Xc???QUp4
zU3{1y9<H0m&3e4yfAalP+uZb<>zi@W)-Mm6{_)r0Y0>INHzqf5oP;oKL(tt}(cX;X
z{m0qt@$peTIyLOCXI5)HgZ%l($EWRPcl+K!Q%afDU)zfh4bS|iu4{hpcl~}f4*QSI
z*V}%)x@o*qw)w>s9m`qMH761O9&Y-(&FVQ>bv4}04%7d1-81F58Mpn#zVG(^)v#aB
zXA>~RTla^ni{-vwjr|%g#+&DmYCfw!9Z!8fTnNIS3At`I>qYxzb=`M3aTxc*ZS{56
zAHQ`+XWQocX1f*B<wyD4?f=<wx_{_*!+QOkdQE?#{jERTj>CP|b;f4>lq~fe_p9CE
z`>?-TwDsc_#lLjkM}^ru=S}B>fBtfIjQO))mh}1aNr;m$U&sEw^F;ZwUHwa|bl>)y
zyB|c-FJZiW2{L&hjK^m_CGh9b>>sdscT{@bczxtP`=87g()$&4!R|k-Q?Ue-k~=q9
z>71iYwz;%g1)YP{{@facWiflxEP-t-#zZS{Fh1t&8dFs9!I<fA^f4Bl1dUK=Un!30
z64)lEOMU|ryZEAujc<svN;-RDoT|7qK?UoyGvQS864->KEof57+Gs^ZP#R)OdP$iS
zqtQh`>p2<Yxrv0Zk^gFh!26(`HUe9lL-BI9Xro=W=kn3ofJY6#dy}+J{uI3=Mu<+w
zR8HZgz)oOP1cenLJ2;Z^A!rd1lyg3$Yy@=V7F|WP2DKJxi6WR7EyNx{nM8vewlcyN
z!Cqx!1N9Kp6r-a-u~*5&5L0E~cO4};Un>(s4QCv@x}HTrK%FpXZ*<6r@OT146%kb~
z#H}bIL&*4pfidVHf@#5+1CO8-A2ZSGRL(YpX9kNj9|-4E@H+U22(dU}G4`;Q2<J6v
zP>oWkGJ>6=D#1w_)RU&NE}a`o${u53AfrIt&|p<0niB#q&8l^Q0F-=ekt(AH4cbyV
ziw49<4~raI3Y{&eqxZ8F)VW-W^wcXV6xKz=RT05i4kN-TQY)Fxmct=};)H0=C8vov
z%Grnts|s^Mi`B!}W;5LiYM~H+ph%*PMBYV(f}<s+DQoI%3$?S#7*Cn1HYZ+FBsNU4
z1<0sOkxleff+mYtuw8~3SPtTUNe<5pZA1k<HlQWKnJa-tmbPqz_hHb;t&$3!ac^RS
z=Bg7E1zBW)VwpCsQY4y0h6_|GE8|IFEoU9VwAch%rQvwVaY9X|JA{D}QRX5Ou4xF-
zSzS-q!6T?Fks@8`741ZbiqdAj(CtfLShW#<w21eGR-@Bkp#QKJRN^_bu^527#~_F!
z)r!(oQ57aU$S@<>02h2qB+Vq2{Kn0!9CJVAEStv|VaS|iuv0xNvupwkMQNLKsV&7o
z1|ubAg;HQTO}Yf5*oFm$B0I^*fLrA<sHDsmwz8UQlLR{1im;-R2aRNGSrIunBOOub
z4rZg&_TwrD&Nu9Qt<h-EKrd?sjf_EA9LdNW()U!^!7CR;1hq0;P9<QBki)boH!p(O
zC-h__Ww}_Ol`6B+P{zfx1a!U_7a(9OGC?WJNyd~|r1rW$>kWhgnYY)?d}-+(%@t+f
zeTA1?bfTB_bqa%W^_kk)FDoCgBxC5oH`jyp%5eFi==3fDW!0MO$nEvodG}gv>l@C?
zYts*Ia=C_MF0{Ec8%c91o;8Q7)t0vt19;Z#hjDsWVHwuvY<iiQ-e67$x)X+$*DbmB
Y-+9HDUB3nAr@+gX`e80_p%)+i0|$YAnE(I)
literal 0
HcmV?d00001
diff --git a/src/containers/camera-modal.jsx b/src/containers/camera-modal.jsx
new file mode 100644
index 000000000..1b7a5de9e
--- /dev/null
+++ b/src/containers/camera-modal.jsx
@@ -0,0 +1,89 @@
+import bindAll from 'lodash.bindall';
+import PropTypes from 'prop-types';
+import React from 'react';
+import {connect} from 'react-redux';
+
+import CameraModalComponent from '../components/camera-modal/camera-modal.jsx';
+import {ModalVideoProvider} from '../lib/camera.js';
+
+import {
+ closeCameraCapture
+} from '../reducers/modals';
+
+class CameraModal extends React.Component {
+ constructor (props) {
+ super(props);
+ bindAll(this, [
+ 'handleCapture',
+ // 'setVideoInput',
+ 'handleSubmit',
+ 'handleCancel',
+ 'setCanvas'
+ // 'enableVideo'
+ ]);
+
+ this.video = null;
+ this.videoDevice = null;
+
+ this.state = {
+ capture: null
+ };
+ }
+ componentWillUnmount () {
+ // const videoDevice = this.props.vm.runtime.ioDevices.video;
+ // videoDevice.disableVideo();
+ this.videoDevice.disableVideo();
+ // this.video = null;
+ }
+ handleCapture () {
+ const capture = this.videoDevice.takeSnapshot();
+ this.setState({capture: capture});
+ }
+ setCanvas (canvas) {
+ this.canvas = canvas;
+ if (this.canvas) {
+ this.videoDevice = new ModalVideoProvider(this.canvas);
+ this.videoDevice.enableVideo();
+ }
+ }
+ handleSubmit () {
+ if (!this.state.capture) return;
+ this.props.onNewCostume(this.state.capture);
+ this.props.onClose();
+ }
+ handleCancel () {
+ this.props.onClose();
+ }
+ render () {
+ return (
+ <CameraModalComponent
+ // vm={this.props.vm}
+ // onBack={this.handleBack}
+ canvasRef={this.setCanvas}
+ capture={this.state.capture}
+ // videoRef={this.setVideoInput}
+ onCancel={this.handleCancel}
+ onCapture={this.handleCapture}
+ onSubmit={this.handleSubmit}
+ />
+ );
+ }
+}
+
+CameraModal.propTypes = {
+ onClose: PropTypes.func,
+ onNewCostume: PropTypes.func
+};
+
+const mapStateToProps = () => ({});
+
+const mapDispatchToProps = dispatch => ({
+ onClose: () => {
+ dispatch(closeCameraCapture());
+ }
+});
+
+export default connect(
+ mapStateToProps,
+ mapDispatchToProps
+)(CameraModal);
diff --git a/src/containers/costume-tab.jsx b/src/containers/costume-tab.jsx
index 6042ceaaa..5146a4f74 100644
--- a/src/containers/costume-tab.jsx
+++ b/src/containers/costume-tab.jsx
@@ -8,12 +8,15 @@ import AssetPanel from '../components/asset-panel/asset-panel.jsx';
import PaintEditorWrapper from './paint-editor-wrapper.jsx';
import CostumeLibrary from './costume-library.jsx';
import BackdropLibrary from './backdrop-library.jsx';
+import CameraModal from './camera-modal.jsx';
import {connect} from 'react-redux';
import {handleFileUpload, costumeUpload} from '../lib/file-uploader.js';
import {
+ closeCameraCapture,
closeCostumeLibrary,
closeBackdropLibrary,
+ openCameraCapture,
openCostumeLibrary,
openBackdropLibrary
} from '../reducers/modals';
@@ -79,6 +82,7 @@ class CostumeTab extends React.Component {
'handleSurpriseBackdrop',
'handleFileUploadClick',
'handleCostumeUpload',
+ 'handleCameraBuffer',
'setFileInput'
]);
const {
@@ -180,6 +184,10 @@ class CostumeTab extends React.Component {
costumeUpload(buffer, fileType, fileName, storage, this.handleNewCostume);
});
}
+ handleCameraBuffer (buffer) {
+ const storage = this.props.vm.runtime.storage;
+ costumeUpload(buffer, 'image/png', 'costume1', storage, this.handleNewCostume);
+ }
handleFileUploadClick () {
this.fileInput.click();
}
@@ -194,11 +202,14 @@ class CostumeTab extends React.Component {
render () {
const {
intl,
+ onNewCostumeFromCameraClick,
onNewLibraryBackdropClick,
onNewLibraryCostumeClick,
backdropLibraryVisible,
+ cameraModalVisible,
costumeLibraryVisible,
onRequestCloseBackdropLibrary,
+ onRequestCloseCameraModal,
onRequestCloseCostumeLibrary,
editingTarget,
sprites,
@@ -234,7 +245,8 @@ class CostumeTab extends React.Component {
},
{
title: intl.formatMessage(messages.addCameraCostumeMsg),
- img: cameraIcon
+ img: cameraIcon,
+ onClick: onNewCostumeFromCameraClick
},
{
title: intl.formatMessage(addFileMessage),
@@ -280,6 +292,13 @@ class CostumeTab extends React.Component {
onRequestClose={onRequestCloseBackdropLibrary}
/>
) : null}
+ {cameraModalVisible ? (
+ <CameraModal
+ vm={vm}
+ onClose={onRequestCloseCameraModal}
+ onNewCostume={this.handleCameraBuffer}
+ />
+ ) : null}
</AssetPanel>
);
}
@@ -287,12 +306,15 @@ class CostumeTab extends React.Component {
CostumeTab.propTypes = {
backdropLibraryVisible: PropTypes.bool,
+ cameraModalVisible: PropTypes.bool,
costumeLibraryVisible: PropTypes.bool,
editingTarget: PropTypes.string,
intl: intlShape,
+ onNewCostumeFromCameraClick: PropTypes.func.isRequired,
onNewLibraryBackdropClick: PropTypes.func.isRequired,
onNewLibraryCostumeClick: PropTypes.func.isRequired,
onRequestCloseBackdropLibrary: PropTypes.func.isRequired,
+ onRequestCloseCameraModal: PropTypes.func.isRequired,
onRequestCloseCostumeLibrary: PropTypes.func.isRequired,
sprites: PropTypes.shape({
id: PropTypes.shape({
@@ -315,6 +337,7 @@ const mapStateToProps = state => ({
editingTarget: state.targets.editingTarget,
sprites: state.targets.sprites,
stage: state.targets.stage,
+ cameraModalVisible: state.modals.cameraCapture,
costumeLibraryVisible: state.modals.costumeLibrary,
backdropLibraryVisible: state.modals.backdropLibrary
});
@@ -328,11 +351,17 @@ const mapDispatchToProps = dispatch => ({
e.preventDefault();
dispatch(openCostumeLibrary());
},
+ onNewCostumeFromCameraClick: () => {
+ dispatch(openCameraCapture());
+ },
onRequestCloseBackdropLibrary: () => {
dispatch(closeBackdropLibrary());
},
onRequestCloseCostumeLibrary: () => {
dispatch(closeCostumeLibrary());
+ },
+ onRequestCloseCameraModal: () => {
+ dispatch(closeCameraCapture());
}
});
diff --git a/src/containers/stage.jsx b/src/containers/stage.jsx
index 96fdf71a6..e6918fdd2 100644
--- a/src/containers/stage.jsx
+++ b/src/containers/stage.jsx
@@ -6,7 +6,7 @@ import VM from 'scratch-vm';
import {connect} from 'react-redux';
import {getEventXY} from '../lib/touch-utils';
-import VideoProvider from '../lib/video/video-provider';
+import {stageVideoProvider} from '../lib/camera';
import StageComponent from '../components/stage/stage.jsx';
@@ -58,7 +58,7 @@ class Stage extends React.Component {
this.renderer = new Renderer(this.canvas);
this.props.vm.attachRenderer(this.renderer);
this.props.vm.runtime.addListener('QUESTION', this.questionListener);
- this.props.vm.setVideoProvider(new VideoProvider());
+ this.props.vm.setVideoProvider(stageVideoProvider(this.props.vm.runtime));
}
shouldComponentUpdate (nextProps, nextState) {
return this.props.width !== nextProps.width ||
diff --git a/src/lib/camera.js b/src/lib/camera.js
new file mode 100644
index 000000000..981ee972d
--- /dev/null
+++ b/src/lib/camera.js
@@ -0,0 +1,654 @@
+import getUserMedia from 'get-user-media-promise';
+import log from './log.js';
+
+class StageVideoProvider {
+ constructor (runtime) {
+ /**
+ * Reference to the owning Runtime.
+ * @type{!Runtime}
+ */
+ this.runtime = runtime;
+
+ /**
+ * Default value for mirrored frames.
+ * @type boolean
+ */
+ this.mirror = true;
+
+ /**
+ * Cache frames for this many ms.
+ * @type number
+ */
+ this._frameCacheTimeout = 16;
+
+ /**
+ * DOM Video element
+ * @private
+ */
+ this._video = null;
+
+ /**
+ * Usermedia stream track
+ * @private
+ */
+ this._track = null;
+
+ /**
+ * Stores some canvas/frame data per resolution/mirror states
+ */
+ this._workspace = [];
+
+ /**
+ * Id representing a Scratch Renderer skin the video is rendered to for
+ * previewing.
+ * @type {number}
+ */
+ this._skinId = -1;
+
+ /**
+ * The Scratch Renderer Skin object.
+ * @type {Skin}
+ */
+ this._skin = null;
+
+ /**
+ * Id for a drawable using the video's skin that will render as a video
+ * preview.
+ * @type {Drawable}
+ */
+ this._drawable = -1;
+
+ /**
+ * Store the last state of the video transparency ghost effect
+ * @type {number}
+ */
+ this._ghost = 0;
+ }
+
+ static get FORMAT_IMAGE_DATA () {
+ return 'image-data';
+ }
+
+ static get FORMAT_CANVAS () {
+ return 'canvas';
+ }
+
+ /**
+ * Dimensions the video stream is analyzed at after its rendered to the
+ * sample canvas.
+ * @type {Array.<number>}
+ */
+ static get DIMENSIONS () {
+ return [480, 360];
+ }
+
+ /**
+ * Order preview drawable is inserted at in the renderer.
+ * @type {number}
+ */
+ static get ORDER () {
+ return 1;
+ }
+
+ /**
+ * Request video be enabled. Sets up video, creates video skin and enables preview.
+ *
+ * ioDevices.video.requestVideo()
+ *
+ * @return {Promise.<Video>} resolves a promise to this IO device when video is ready.
+ */
+ enableVideo () {
+ this.enabled = true;
+ return this._setupVideo();
+ }
+
+ /**
+ * Disable video stream (turn video off)
+ */
+ disableVideo () {
+ this.enabled = false;
+ // If we have begun a setup process, call _teardown after it completes
+ if (this._singleSetup) {
+ this._singleSetup
+ .then(this._teardown.bind(this))
+ .catch(err => this.onError(err));
+ }
+ }
+
+ /**
+ * async part of disableVideo
+ * @private
+ */
+ _teardown () {
+ // we might be asked to re-enable before _teardown is called, just ignore it.
+ if (this.enabled === false) {
+ this._disablePreview();
+ this._singleSetup = null;
+ // by clearing refs to video and track, we should lose our hold over the camera
+ this._video = null;
+ if (this._track) {
+ this._track.stop();
+ }
+ this._track = null;
+ }
+ }
+
+ /**
+ * Return frame data from the video feed in a specified dimensions, format, and mirroring.
+ *
+ * @param {object} frameInfo A descriptor of the frame you would like to receive.
+ * @param {Array.<number>} frameInfo.dimensions [width, height] array of numbers. Defaults to [480,360]
+ * @param {boolean} frameInfo.mirror If you specificly want a mirror/non-mirror frame, defaults to the global
+ * mirror state (ioDevices.video.mirror)
+ * @param {string} frameInfo.format Requested video format, available formats are 'image-data' and 'canvas'.
+ * @param {number} frameInfo.cacheTimeout Will reuse previous image data if the time since capture is less than
+ * the cacheTimeout. Defaults to 16ms.
+ *
+ * @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors.
+ */
+ getFrame ({
+ dimensions = StageVideoProvider.DIMENSIONS,
+ mirror = this.mirror,
+ format = StageVideoProvider.FORMAT_IMAGE_DATA,
+ cacheTimeout = this._frameCacheTimeout
+ }) {
+ if (!this.videoReady) {
+ return null;
+ }
+ const [width, height] = dimensions;
+ const workspace = this._getWorkspace({dimensions, mirror: Boolean(mirror)});
+ const {videoWidth, videoHeight} = this._video;
+ const {canvas, context, lastUpdate, cacheData} = workspace;
+ const now = Date.now();
+
+ // if the canvas hasn't been updated...
+ if (lastUpdate + cacheTimeout < now) {
+
+ if (mirror) {
+ context.scale(-1, 1);
+ context.translate(width * -1, 0);
+ }
+
+ context.drawImage(this._video,
+ // source x, y, width, height
+ 0, 0, videoWidth, videoHeight,
+ // dest x, y, width, height
+ 0, 0, width, height
+ );
+
+ context.resetTransform();
+ workspace.lastUpdate = now;
+ }
+
+ // each data type has it's own data cache, but the canvas is the same
+ if (!cacheData[format]) {
+ cacheData[format] = {lastUpdate: 0};
+ }
+ const formatCache = cacheData[format];
+
+ if (formatCache.lastUpdate + cacheTimeout < now) {
+ if (format === StageVideoProvider.FORMAT_IMAGE_DATA) {
+ formatCache.lastData = context.getImageData(0, 0, width, height);
+ } else if (format === StageVideoProvider.FORMAT_CANVAS) {
+ // this will never change
+ formatCache.lastUpdate = Infinity;
+ formatCache.lastData = canvas;
+ } else {
+ log.error(`video io error - unimplemented format ${format}`);
+ // cache the null result forever, don't log about it again..
+ formatCache.lastUpdate = Infinity;
+ formatCache.lastData = null;
+ }
+
+ // rather than set to now, this data is as stale as it's canvas is
+ formatCache.lastUpdate = Math.max(workspace.lastUpdate, formatCache.lastUpdate);
+ }
+
+ return formatCache.lastData;
+ }
+
+ /**
+ * Set the preview ghost effect
+ * @param {number} ghost from 0 (visible) to 100 (invisible) - ghost effect
+ */
+ setPreviewGhost (ghost) {
+ this._ghost = ghost;
+ if (this._drawable) {
+ this.runtime.renderer.updateDrawableProperties(this._drawable, {ghost});
+ }
+ }
+
+ /**
+ * Method called when an error happens. Default implementation is just to log error.
+ *
+ * @abstract
+ * @param {Error} error An error object from getUserMedia or other source of error.
+ */
+ onError (error) {
+ log.error('Unhandled video io device error', error);
+ }
+
+ /**
+ * Create a video stream.
+ * Should probably be moved to -render or somewhere similar later
+ * @private
+ * @return {Promise} When video has been received, rejected if video is not received
+ */
+ _setupVideo () {
+ // We cache the result of this setup so that we can only ever have a single
+ // video/getUserMedia request happen at a time.
+ if (this._singleSetup) {
+ return this._singleSetup;
+ }
+
+ this._singleSetup = getUserMedia({ // navigator.mediaDevices.getUserMedia({
+ audio: false,
+ video: {
+ width: {min: 480, ideal: 640},
+ height: {min: 360, ideal: 480}
+ }
+ })
+ .then(stream => {
+ this._video = document.createElement('video');
+ // Use the new srcObject API, falling back to createObjectURL
+ try {
+ this._video.srcObject = stream;
+ } catch (error) {
+ this._video.src = window.URL.createObjectURL(stream);
+ }
+ // Hint to the stream that it should load. A standard way to do this
+ // is add the video tag to the DOM. Since this extension wants to
+ // hide the video tag and instead render a sample of the stream into
+ // the webgl rendered Scratch canvas, another hint like this one is
+ // needed.
+ this._video.play(); // Needed for Safari/Firefox, Chrome auto-plays.
+ this._track = stream.getTracks()[0];
+ this._setupPreview();
+ return this;
+ })
+ .catch(error => {
+ this._singleSetup = null;
+ this.onError(error);
+ });
+
+ return this._singleSetup;
+ }
+
+ _disablePreview () {
+ if (this._skin) {
+ this._skin.clear();
+ this.runtime.renderer.updateDrawableProperties(this._drawable, {visible: false});
+ }
+ this._renderPreviewFrame = null;
+ }
+
+ _setupPreview () {
+ const {renderer} = this.runtime;
+ if (!renderer) return;
+
+ if (this._skinId === -1 && this._skin === null && this._drawable === -1) {
+ this._skinId = renderer.createPenSkin();
+ this._skin = renderer._allSkins[this._skinId];
+ this._drawable = renderer.createDrawable();
+ renderer.setDrawableOrder(
+ this._drawable,
+ StageVideoProvider.ORDER
+ );
+ renderer.updateDrawableProperties(this._drawable, {
+ skinId: this._skinId
+ });
+ }
+
+ // if we haven't already created and started a preview frame render loop, do so
+ if (!this._renderPreviewFrame) {
+ renderer.updateDrawableProperties(this._drawable, {
+ ghost: this._ghost,
+ visible: true
+ });
+
+ this._renderPreviewFrame = () => {
+ clearTimeout(this._renderPreviewTimeout);
+ if (!this._renderPreviewFrame) {
+ return;
+ }
+
+ this._renderPreviewTimeout = setTimeout(this._renderPreviewFrame, this.runtime.currentStepTime);
+
+ const canvas = this.getFrame({format: StageVideoProvider.FORMAT_CANVAS});
+
+ if (!canvas) {
+ this._skin.clear();
+ return;
+ }
+
+ const xOffset = StageVideoProvider.DIMENSIONS[0] / -2;
+ const yOffset = StageVideoProvider.DIMENSIONS[1] / 2;
+ this._skin.drawStamp(canvas, xOffset, yOffset);
+ this.runtime.requestRedraw();
+ };
+
+ this._renderPreviewFrame();
+ }
+ }
+
+ get videoReady () {
+ if (!this.enabled) {
+ return false;
+ }
+ if (!this._video) {
+ return false;
+ }
+ if (!this._track) {
+ return false;
+ }
+ const {videoWidth, videoHeight} = this._video;
+ if (typeof videoWidth !== 'number' || typeof videoHeight !== 'number') {
+ return false;
+ }
+ if (videoWidth === 0 || videoHeight === 0) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * get an internal workspace for canvas/context/caches
+ * this uses some document stuff to create a canvas and what not, probably needs abstraction
+ * into the renderer layer?
+ * @private
+ * @return {object} A workspace for canvas/data storage. Internal format not documented intentionally
+ */
+ _getWorkspace ({dimensions, mirror}) {
+ let workspace = this._workspace.find(space => (
+ space.dimensions.join('-') === dimensions.join('-') &&
+ space.mirror === mirror
+ ));
+ if (!workspace) {
+ workspace = {
+ dimensions,
+ mirror,
+ canvas: document.createElement('canvas'),
+ lastUpdate: 0,
+ cacheData: {}
+ };
+ workspace.canvas.width = dimensions[0];
+ workspace.canvas.height = dimensions[1];
+ workspace.context = workspace.canvas.getContext('2d');
+ this._workspace.push(workspace);
+ }
+ return workspace;
+ }
+}
+
+class ModalVideoProvider {
+ constructor (canvas) {
+ /**
+ * Captured image data
+ */
+ this._capture = null;
+
+ /**
+ * Cache frames for this many ms.
+ * @type number
+ */
+ this._frameCacheTimeout = 16;
+
+ this._canvas = canvas;
+
+ this._video = null;
+
+ // this._workspace = [];
+
+ /**
+ * Usermedia stream track
+ * @private
+ */
+ this._track = null;
+ }
+
+ static get FORMAT_IMAGE_DATA () {
+ return 'image-data';
+ }
+
+ static get FORMAT_CANVAS () {
+ return 'canvas';
+ }
+
+ /**
+ * Dimensions the video stream is analyzed at after its rendered to the
+ * sample canvas.
+ * @type {Array.<number>}
+ */
+ static get DIMENSIONS () {
+ return [480, 360];
+ }
+
+ setVideo (video) {
+ this._video = video;
+ }
+
+ get videoReady () {
+ // if (!this.enabled) {
+ // return false;
+ // }
+ if (!this._video) {
+ return false;
+ }
+ if (!this._track) {
+ return false;
+ }
+ const {videoWidth, videoHeight} = this._video;
+ if (typeof videoWidth !== 'number' || typeof videoHeight !== 'number') {
+ return false;
+ }
+ if (videoWidth === 0 || videoHeight === 0) {
+ return false;
+ }
+ return true;
+ }
+
+ enableVideo () {
+ const thisContext = this;
+ this._video = this._video ? this._video : document.createElement('video');
+ // TODO possibly make common function for this
+ getUserMedia({
+ audio: false,
+ video: true
+ })
+ .then(userMediaStream => {
+ try {
+ thisContext._video.srcObject = userMediaStream;
+ } catch (e) {
+ thisContext._video.src = window.URL.createObjectURL(userMediaStream);
+ }
+ thisContext._track = userMediaStream.getTracks()[0];
+
+ const width = 960; // abstract this out
+ const height = 720;
+
+ const ctx = thisContext._canvas.getContext('2d');
+
+ ctx.scale(-1, 1);
+ ctx.translate(width * -1, 0);
+
+ thisContext._videoFeedInterval = setInterval(() => ctx.drawImage(thisContext._video,
+ // source x, y, width, height
+ 0, 0, thisContext._video.videoWidth, thisContext._video.videoHeight,
+ // dest x, y, width, height
+ 0, 0, width, height
+ ), thisContext._frameCacheTimeout);
+
+ // The following also works...
+ // thisContext._videoFeedInterval = setInterval(
+ // () => this.getFrame({format: ModalVideoProvider.FORMAT_CANVAS}),
+ // thisContext._frameCacheTimeout);
+
+
+ // thisContext._setupPreview();
+ })
+ .catch(e => {
+ log.warn(e); // TODO make common function for this
+ });
+ }
+
+ // _setupPreview () {
+ //
+ // // if we haven't already created and started a preview frame render loop, do so
+ // if (!this._renderPreviewFrame) {
+ //
+ // this._renderPreviewFrame = () => {
+ // clearTimeout(this._renderPreviewTimeout);
+ // if (!this._renderPreviewFrame) {
+ // return;
+ // }
+ //
+ // this._renderPreviewTimeout = setTimeout(this._renderPreviewFrame, this._frameCacheTimeout);
+ //
+ // const canvas = this.getFrame({format: ModalVideoProvider.FORMAT_CANVAS});
+ //
+ // if (!canvas) {
+ // // this._skin.clear();
+ // return;
+ // }
+ //
+ // // const xOffset = ModalVideoProvider.DIMENSIONS[0] / -2;
+ // // const yOffset = ModalVideoProvider.DIMENSIONS[1] / 2;
+ // // this._skin.drawStamp(canvas, xOffset, yOffset);
+ // // this.runtime.requestRedraw();
+ // };
+ //
+ // this._renderPreviewFrame();
+ // }
+ // }
+
+ takeSnapshot () {
+ clearInterval(this._videoFeedInterval);
+
+ // clearTimeout(this._renderPreviewTimeout);
+
+ return this._canvas.toDataURL('image/png');
+ }
+
+ getSnapshot () {
+ return this._capture;
+ }
+
+ /**
+ * get an internal workspace for canvas/context/caches
+ * this uses some document stuff to create a canvas and what not, probably needs abstraction
+ * into the renderer layer?
+ * @private
+ * @return {object} A workspace for canvas/data storage. Internal format not documented intentionally
+ */
+ // _getWorkspace ({dimensions, mirror}) {
+ // let workspace = this._workspace.find(space => (
+ // space.dimensions.join('-') === dimensions.join('-') &&
+ // space.mirror === mirror
+ // ));
+ // if (!workspace) {
+ // workspace = {
+ // dimensions,
+ // mirror,
+ // canvas: this._canvas, // document.createElement('canvas'),
+ // lastUpdate: 0,
+ // cacheData: {}
+ // };
+ // workspace.canvas.width = dimensions[0];
+ // workspace.canvas.height = dimensions[1];
+ // workspace.context = workspace.canvas.getContext('2d');
+ // this._workspace.push(workspace);
+ // }
+ // return workspace;
+ // }
+
+ /**
+ * Return frame data from the video feed in a specified dimensions, format, and mirroring.
+ *
+ * @param {object} frameInfo A descriptor of the frame you would like to receive.
+ * @param {Array.<number>} frameInfo.dimensions [width, height] array of numbers. Defaults to [480,360]
+ * @param {boolean} frameInfo.mirror If you specificly want a mirror/non-mirror frame, defaults to the global
+ * mirror state (ioDevices.video.mirror)
+ * @param {string} frameInfo.format Requested video format, available formats are 'image-data' and 'canvas'.
+ * @param {number} frameInfo.cacheTimeout Will reuse previous image data if the time since capture is less than
+ * the cacheTimeout. Defaults to 16ms.
+ *
+ * @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors.
+ */
+ // getFrame ({
+ // dimensions = ModalVideoProvider.DIMENSIONS,
+ // mirror = this.mirror,
+ // format = ModalVideoProvider.FORMAT_IMAGE_DATA,
+ // cacheTimeout = this._frameCacheTimeout
+ // }) {
+ // if (!this.videoReady) {
+ // return null;
+ // }
+ // const [width, height] = dimensions;
+ // const workspace = this._getWorkspace({dimensions, mirror: Boolean(mirror)});
+ // const {videoWidth, videoHeight} = this._video;
+ // const {canvas, context, lastUpdate, cacheData} = workspace;
+ // const now = Date.now();
+ //
+ // // if the canvas hasn't been updated...
+ // if (lastUpdate + cacheTimeout < now) {
+ //
+ // if (mirror) {
+ // context.scale(-1, 1);
+ // context.translate(width * -1, 0);
+ // }
+ //
+ // context.drawImage(this._video,
+ // // source x, y, width, height
+ // 0, 0, videoWidth, videoHeight,
+ // // dest x, y, width, height
+ // 0, 0, width, height
+ // );
+ //
+ // context.resetTransform();
+ // workspace.lastUpdate = now;
+ // }
+ //
+ // // each data type has it's own data cache, but the canvas is the same
+ // if (!cacheData[format]) {
+ // cacheData[format] = {lastUpdate: 0};
+ // }
+ // const formatCache = cacheData[format];
+ //
+ // if (formatCache.lastUpdate + cacheTimeout < now) {
+ // if (format === ModalVideoProvider.FORMAT_IMAGE_DATA) {
+ // formatCache.lastData = context.getImageData(0, 0, width, height);
+ // } else if (format === ModalVideoProvider.FORMAT_CANVAS) {
+ // // this will never change
+ // formatCache.lastUpdate = Infinity;
+ // formatCache.lastData = canvas;
+ // } else {
+ // log.error(`video io error - unimplemented format ${format}`);
+ // // cache the null result forever, don't log about it again..
+ // formatCache.lastUpdate = Infinity;
+ // formatCache.lastData = null;
+ // }
+ //
+ // // rather than set to now, this data is as stale as it's canvas is
+ // formatCache.lastUpdate = Math.max(workspace.lastUpdate, formatCache.lastUpdate);
+ // }
+ //
+ // return formatCache.lastData;
+ // }
+
+ disableVideo () {
+ if (this._video) {
+ if (this._track) this._track.stop();
+ this._video.pause();
+ this._video.srcObject = null;
+ }
+ }
+}
+
+const stageVideoProvider = function (runtime) {
+ return new StageVideoProvider(runtime);
+};
+
+export {
+ stageVideoProvider,
+ ModalVideoProvider
+};
diff --git a/src/lib/file-uploader.js b/src/lib/file-uploader.js
index 040a026b2..ca4ce8773 100644
--- a/src/lib/file-uploader.js
+++ b/src/lib/file-uploader.js
@@ -77,7 +77,8 @@ const cacheAsset = function (storage, fileName, assetType, dataFormat, data) {
/**
* Handles loading a costume or a backdrop using the provided, context-relevant information.
- * @param {ArrayBuffer} fileData The costume data to load
+ * @param {ArrayBuffer | string} fileData The costume data to load (this can be an image url
+ * iff the image is a bitmap)
* @param {string} fileType The MIME type of this file
* @param {string} costumeName The user-readable name to use for the costume.
* @param {ScratchStorage} storage The ScratchStorage instance to cache the costume data
diff --git a/src/reducers/modals.js b/src/reducers/modals.js
index 481c44f29..c373ce796 100644
--- a/src/reducers/modals.js
+++ b/src/reducers/modals.js
@@ -4,6 +4,7 @@ const OPEN_MODAL = 'scratch-gui/modals/OPEN_MODAL';
const CLOSE_MODAL = 'scratch-gui/modals/CLOSE_MODAL';
const MODAL_BACKDROP_LIBRARY = 'backdropLibrary';
+const MODAL_CAMERA_CAPTURE = 'cameraCapture';
const MODAL_COSTUME_LIBRARY = 'costumeLibrary';
const MODAL_EXTENSION_LIBRARY = 'extensionLibrary';
const MODAL_IMPORT_INFO = 'importInfo';
@@ -18,6 +19,7 @@ const MODAL_TIPS_LIBRARY = 'tipsLibrary';
const initialState = {
[MODAL_BACKDROP_LIBRARY]: false,
+ [MODAL_CAMERA_CAPTURE]: false,
[MODAL_COSTUME_LIBRARY]: false,
[MODAL_EXTENSION_LIBRARY]: false,
[MODAL_IMPORT_INFO]: false,
@@ -60,6 +62,10 @@ const openBackdropLibrary = function () {
analytics.pageview('/libraries/backdrops');
return openModal(MODAL_BACKDROP_LIBRARY);
};
+const openCameraCapture = function () {
+ analytics.pageview('/modals/camera');
+ return openModal(MODAL_CAMERA_CAPTURE);
+};
const openCostumeLibrary = function () {
analytics.pageview('/libraries/costumes');
return openModal(MODAL_COSTUME_LIBRARY);
@@ -99,6 +105,9 @@ const openTipsLibrary = function () {
const closeBackdropLibrary = function () {
return closeModal(MODAL_BACKDROP_LIBRARY);
};
+const closeCameraCapture = function () {
+ return closeModal(MODAL_CAMERA_CAPTURE);
+};
const closeCostumeLibrary = function () {
return closeModal(MODAL_COSTUME_LIBRARY);
};
@@ -129,6 +138,7 @@ const closeTipsLibrary = function () {
export {
reducer as default,
openBackdropLibrary,
+ openCameraCapture,
openCostumeLibrary,
openExtensionLibrary,
openImportInfo,
@@ -139,6 +149,7 @@ export {
openSoundRecorder,
openTipsLibrary,
closeBackdropLibrary,
+ closeCameraCapture,
closeCostumeLibrary,
closeExtensionLibrary,
closeImportInfo,
--
GitLab