Browse Source

Add bigquery-import sample.

ryanpbrewster-patch-1
Nicolas Garnier 9 years ago
parent
commit
557d006393
  1. 6
      README.md
  2. 41
      bigquery-import/README.md
  3. 10
      bigquery-import/env.json
  4. 63
      bigquery-import/index.js
  5. 7
      bigquery-import/package.json

6
README.md

@ -36,11 +36,11 @@ Integrates the Google Translate API to perform automatic text translation across
Make sure to keep the number of child nodes below a certain threshold. For instance this can be useful to limit the number of lines of logs or chat history below a given number. Make sure to keep the number of child nodes below a certain threshold. For instance this can be useful to limit the number of lines of logs or chat history below a given number.
### [Full-text search via Algolia](/fulltext-search) TODO import from go/hearth-codesamples ### [Import Data to Big Query](/bigquery-import)
... Copies Firebase Database elements into BigQuery automatically. This can be useful for instance for logs analysis.
### [Import Data to Big Query](/fulltext-search) TODO import from go/hearth-codesamples ### [Full-text search via Algolia](/fulltext-search) TODO import from go/hearth-codesamples
... ...

41
bigquery-import/README.md

@ -0,0 +1,41 @@
# Import Data to Big Query
This template shows how to keep the number of child nodes in a Firebase database below a given number. This can be used to limit the number of lines of a chat history or logs.
## Cloud Function Code
See file [index.js](index.js) for the code.
The dependencies are listed in [package.json](package.json).
## Sample Database Structure
As an example we'll be using a simple logs database structure:
```
/functions-project-12345
/logs
/key-123456
text: "User signed in."
/key-123457
text: "Error: Could not connect to Database"
```
## Trigger rules
Below is the trigger rule for the `addtobigquery` function making sure it's triggered when a new log entry is added.
```
"functions": {
".source": "functions",
"addtobigquery": {
"triggers": {
"database": {
"path": "/logs/$logid"
}
}
}
}
```

10
bigquery-import/env.json

@ -0,0 +1,10 @@
// This file should be placed at the root of you project: the same directory as your firebase.json
// file, not your functions source directory.
// TODO: Input your Google Project ID, Private key and Email below.
{
"google": {
"project_id": "<MY_GOOGLE_PROJECT_ID>",
"private_key": "<MY_GOOGLE_PROJECT_PRIVATE_KEY>",
"client_email": "<MY_GOOGLE_PROJECT_CLIENT_EMAIL>"
}
}

63
bigquery-import/index.js

@ -0,0 +1,63 @@
/**
* Copyright 2015 Google Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
// Create an all access Firebase Database reference.
var Firebase = require('firebase');
var env = require('./env');
var ref = new Firebase(env.get('firebase.database.url'), 'admin');
ref.auth(env.get('firebase.database.token'));
// Authenticate to gcloud.
// TODO: Make sure you add your Google Project ID, Private key and Email into env.json
var gcloudconfig = {
projectId: env.get('google.project_id'),
credentials: {
private_key: env.get('google.private_key'),
client_email: env.get('google.client_email')
}
};
var gcloud = require('gcloud')(gcloudconfig);
var bigquery = gcloud.bigquery();
// TODO: Change <YOUR-DATASET-NAME> with your BigQuery dataset name.
var dataset = bigquery.dataset('<YOUR-DATASET-NAME>');
// TODO: Change <YOUR-TABLE-NAME> with your BigQuery table name.
var table = dataset.table('<YOUR-TABLE-NAME>');
// Copies the Firebase Database element to BigQuery
function addtobigquery(context, data) {
ref.child(data.path).once('value', function (snap) {
table.insert({
ID: snap.key(),
MESSAGE: snap.val().message,
NUMBER: snap.val().number
}, function(err, insertErr, resp) {
if (err) {
console.log(err);
context.done(err);
} else if (insertErr) {
console.log(insertErr);
context.done(insertErr);
} else {
context.done();
}
});
});
}
module.exports = {
addtobigquery: addtobigquery
}

7
bigquery-import/package.json

@ -0,0 +1,7 @@
{
"main": "index.js",
"dependencies": {
"firebase": "^2.4.0",
"gcloud": "^0.27.0"
}
}
Loading…
Cancel
Save