Merge pull request #963 from mozilla/gcs

WIP: GCS

package.json

@@ -130,6 +130,7 @@
     "webpack-unassert-loader": "^1.2.0"
   },
   "dependencies": {
+    "@google-cloud/storage": "^2.0.3",
     "aws-sdk": "^2.328.0",
     "babel-polyfill": "^6.26.0",
     "choo": "^6.12.1",
@@ -137,6 +138,7 @@
     "convict": "^4.4.0",
     "express": "^4.16.3",
     "express-ws": "^4.0.0",
+    "fast-crc32c": "^1.0.4",
     "fluent": "^0.8.1",
     "fluent-langneg": "^0.1.0",
     "helmet": "^3.13.0",

server/config.js

@@ -9,6 +9,11 @@ const conf = convict({
     default: '',
     env: 'S3_BUCKET'
   },
+  gcs_bucket: {
+    format: String,
+    default: '',
+    env: 'GCS_BUCKET'
+  },
   expire_times_seconds: {
     format: Array,
     default: [300, 3600, 86400, 604800],

server/storage/gcs.js

@@ -0,0 +1,37 @@
+const { Storage } = require('@google-cloud/storage');
+const storage = new Storage();
+
+class GCSStorage {
+  constructor(config, log) {
+    this.bucket = storage.bucket(config.gcs_bucket);
+    this.log = log;
+  }
+
+  async length(id) {
+    const data = await this.bucket.file(id).getMetadata();
+    return data[0].size;
+  }
+
+  getStream(id) {
+    return this.bucket.file(id).createReadStream();
+  }
+
+  set(id, file) {
+    return new Promise((resolve, reject) => {
+      file
+        .pipe(this.bucket.file(id).createWriteStream())
+        .on('error', reject)
+        .on('finish', resolve);
+    });
+  }
+
+  del(id) {
+    return this.bucket.file(id).delete();
+  }
+
+  ping() {
+    return this.bucket.exists();
+  }
+}
+
+module.exports = GCSStorage;

server/storage/index.js

@@ -9,7 +9,14 @@ function getPrefix(seconds) {
 
 class DB {
   constructor(config) {
-    const Storage = config.s3_bucket ? require('./s3') : require('./fs');
+    let Storage = null;
+    if (config.s3_bucket) {
+      Storage = require('./s3');
+    } else if (config.gcs_bucket) {
+      Storage = require('./gcs');
+    } else {
+      Storage = require('./fs');
+    }
     this.log = mozlog('send.storage');
 
     this.storage = new Storage(config, this.log);