ert78gb / google-pubsub-emulator Goto Github PK
View Code? Open in Web Editor NEWGoogle PubSub Emulator wrapper to nodejs
Google PubSub Emulator wrapper to nodejs
With the latest @google-cloud/[email protected], I got
TypeError: PubSub is not a constructor
at PubSubEmulator._getPubSubClient (node_modules/google-pubsub-emulator/src/pubsub-emulator.js:292:22)
at PubSubEmulator._createTopics (node_modules/google-pubsub-emulator/src/pubsub-emulator.js:302:25)
at PubSubEmulator.startSuccessListener (node_modules/google-pubsub-emulator/src/pubsub-emulator.js:59:18)
at PubSubEmulator._setState (node_modules/google-pubsub-emulator/src/pubsub-emulator.js:207:24)
at PubSubEmulator._processStd (node_modules/google-pubsub-emulator/src/pubsub-emulator.js:154:12)
at PubSubEmulator._emulatorStdErrListener (node_modules/google-pubsub-emulator/src/pubsub-emulator.js:253:10)
at addChunk (_stream_readable.js:263:12)
at readableAddChunk (_stream_readable.js:250:11)
at Socket.Readable.push (_stream_readable.js:208:10)
at Pipe.onread (net.js:594:20)
When I changed the "src/pubsub-emulator.js"
from
290 const PubSub = require('@google-cloud/pubsub').PubSub
to
290 const PubSub = require('@google-cloud/pubsub')
The message went away. Would you please take a look ?
Even when trying to stop the emulator manually on exit the Java servers will be left running in the background resulting in the following error the next time:
Exception in thread "main" java.net.BindException: Address already in use
Listing pubsub as a dependency locks the dependent project to that version unless they want multiple versions installed which doesn't end well since google relies heavily on grpc
Hi,
I have a module below that encapsulates all my pubsub logic.
When my app starts it calls init() below which sets up all the topics and subscribers. This is fine if I deploy to gcp, however .. because google pubsub cannot push even to 'ngrok'ed endpoints runnign on my local machine obviously I need another solution - so I am trying your google-pubsub-emulator
I installed the pub sub emulator and can start the emulator in my code below..
but how can I tweak my code so that my calls to getTopics and pubsubClient.createTopic go via the simulator ? so confused...
'use strict'
const _ = require('underscore')
const async = require('async')
var config = require('../../../../config')
const logger = require('./bunyan').logger
const PubSub = require('@google-cloud/pubsub')
const Emulator = require('google-pubsub-emulator')
const projectId = 'xxx-communication-service'
const pubsubClient = PubSub({
projectId: projectId
})
var prefix = `projects/${projectId}/topics/`
function init (callback) {
createQueues(callback)
}
function createQueues (parentCallback) {
var topicsNames = []
var subscriptionNames = []
async.waterfall([
function (callback) {
process.env.GCLOUD_PROJECT = projectId
const options = {
debug: true, // if you like to see the emulator output
topics: [
'projects/project-id/topics/topic-1' // automatically created topic
]
}
var emulator = new Emulator(options)
emulator.start().then(() => {
console.log(`emulator started.`)
callback()
})
.catch((err) => {
console.error('ERROR:', err)
callback(err)
})
},
function (callback) {
//get all topics already existing
pubsubClient.getTopics(function (err, topics) {
if (!err) {
topicsNames = _.union(topicsNames, _.pluck(topics, 'name'))
// logger.info('got topics ' + JSON.stringify(topicsNames))
callback(null)
} else {
callback(err)
}
})
},
function (callback) {
//get all subscriptions already existing
pubsubClient.getSubscriptions(function (err, subscriptions) {
if (!err) {
logger.info('got subscriptions ' + JSON.stringify(subscriptions))
subscriptionNames = _.union(subscriptions, _.pluck(subscriptions, 'name'))
// logger.info('got subscriptions ' + JSON.stringify(subscriptionNames))
callback(null)
} else {
callback(err)
}
})
},
function (callback) {
//set up poison pill topic which will be pull
var topic = 'error'
if (!_.contains(topicsNames, `${prefix}${topic}-${config['PUBSUB_ENV']}`)) {
pubsubClient.createTopic(`${topic}-${config['PUBSUB_ENV']}`)
.then((results) => {
const topic = results[0]
console.log(`Topic ${topic.name} created.`)
callback()
})
.catch((err) => {
console.error('ERROR:', err)
callback(err)
})
} else {
callback()
}
}, function (callback) {
//set up email topic if not existing
var topic = 'email-job'
if (!_.contains(topicsNames, `${prefix}${topic}-${config['PUBSUB_ENV']}`)) {
pubsubClient.createTopic(`${topic}-${config['PUBSUB_ENV']}`)
.then((results) => {
const topic = results[0]
console.log(`Topic ${topic.name} created.`)
callback()
})
.catch((err) => {
console.error('ERROR:', err)
callback(err)
})
} else {
callback()
}
}, function (callback) {
//set up subscriber to email topic if not existing
// var subscriber = 'email-job-subscriber'
// if (!_.contains(subscriptionNames, `${prefix}${subscriber}-${config['PUBSUB_ENV']}`)) {
// callback()
// } else {
// callback()
// }
}], function (err, result) {
if (err) {
console.log(JSON.stringify(err))
parentCallback(err)
} else {
parentCallback()
}
})
}
module.exports = {
init: init
}
Hi - Has anyone experienced the issue of the emulator running out of memory after the upgrade to @google-cloud/pubsub 3.0.0? We use the pubsub emulator to run our integration tests with a docker image set up:
Build-agent version 1.0.167651-d52a9c2e (2023-04-18T11:21:26+0000)
System information:
Server Version: 20.10.18
Storage Driver: overlay2
Backing Filesystem: xfs
Cgroup Driver: cgroupfs
Cgroup Version: 1
Kernel Version: 5.15.0-1030-aws
Operating System: Ubuntu 20.04.5 LTS
OSType: linux
Architecture: x86_64
Starting container hiveops/circleci-deploy:16.14
Our resources run out almost immediately originally when setting up the emulator but even after we doubled our resources it runs out of memory during publishing:
google-pubsub-emulator/package.json
Lines 46 to 48 in 5f9f4ff
"peerDependencies": {
"@google-cloud/pubsub": ">=2.16.0 <4.0.0"
}
Version 4 several weeks ago. Thanks.
Description
When executing await pubsubEmulator.start();
on my machine, the process is stuck.
After investigation, the event "EmulatorStates.RUNNING" is not emitted because the value of PUBSUB_EMULATOR_RUNNING_KEY is not the same as the expected one.
I got: [pubsub] INFOS: Server started, listening on
The expected value is : [pubsub] INFO: Server started, listening on
.
Possible solution
Log:
stderr: DEBUG: Running [gcloud.beta.emulators.pubsub.start] with arguments: [--host-port: "<googlecloudsdk.calliope.arg_parsers.HostPort object at 0x7ffa961d4650>", --log-http: "true", --user-output-enabled: "true", --verbosity: "debug"]
stderr: DEBUG: Found Cloud SDK root: /home/oc/google-cloud-sdk-215.0.0-linux-x86_64/google-cloud-sdk
stderr: Executing: /home/oc/google-cloud-sdk-215.0.0-linux-x86_64/google-cloud-sdk/platform/pubsub-emulator/bin/cloud-pubsub-emulator --host=localhost --port=12202
stderr: [pubsub] This is the Google Pub/Sub fake.
stderr: [pubsub] Implementation may be incomplete or differ from the real system.
stderr: [pubsub] mai 30, 2019 11:12:10 AM com.google.cloud.pubsub.testing.v1.Main main
stderr: [pubsub] INFOS: IAM integration is disabled. IAM policy methods and ACL checks are not supported
stderr: [pubsub] mai 30, 2019 11:12:11 AM io.gapi.emulators.netty.NettyUtil applyJava7LongHostnameWorkaround
stderr: [pubsub] INFOS: Applied Java 7 long hostname workaround.
stderr: [pubsub] mai 30, 2019 11:12:11 AM com.google.cloud.pubsub.testing.v1.Main main
stderr: [pubsub] INFOS: Server started, listening on 12202
Environment: Ubuntu 18.04
Gcloud tools installed:
Tool | Version |
---|---|
Google Cloud SDK | 248.0.0 |
beta | 2019.05.17 |
bq | 2.0.43 |
cloud-datastore-emulator | 2.1.0 |
core | 2019.05.24 |
gsutil | 4.38 |
pubsub-emulator | 2019.04.26 |
Dependencies in the project:
"@google-cloud/pubsub": "^0.28.1",
Dev-dependencies:
"google-pubsub-emulator": "^3.0.0",
A declarative, efficient, and flexible JavaScript library for building user interfaces.
๐ Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. ๐๐๐
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google โค๏ธ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.