with docker-compose provider db && > python 3.5 install && use virtualenv
- install virtualenv
use pip3
pip3 install virtualenv
or:
pip install virtualenv
- create mysql tap virtualenv && install python packages
virtualenv mysql
source ./mysql/bin/activate
pip install tap-mysql
- create postgres tap virtualenv && install python packages
virtualenv postgres
source ./postgres/bin/activate
pip install target-postgres
- init some mysql data
docker-compose contains one git server gogs && start && config it
- mysql tap config
tap.json
{
"host": "localhost",
"port": "3306",
"user": "root",
"password": "dalongrong"
}
- postgres target config
target.json
{
"host": "localhost",
"port": 5432,
"dbname": "postgres",
"user": "postgres",
"password": "postgres",
"schema": "public"
}
- discover tables && generate json file
./mysql/bin/tap-mysql --config config.json --discover > properties.json
- add sync table conf (with full table)
search for stream
: repository
,
"breadcrumb": [],
"metadata": {
+ "selected-by-default": true,
"database-name": "gogs",
"row-count": 1,
"is-view": false,
+ "selected": true,
+ "replication-method": "FULL_TABLE",
- run sync
./mysql/bin/tap-mysql --config tap.json --properties properties.json | ./postgres/bin/target-postgres --config ta
rget.json
- some result
./mysql/bin/tap-mysql --config tap.json --properties properties.json | ./postgres/bin/target-postgres --config target.json
INFO Server Parameters: version: 5.7.16, wait_timeout: 2700, innodb_lock_wait_timeout: 2700, max_allowed_packet: 4194304, interactive_timeout: 28800
INFO Server SSL Parameters (blank means SSL is not active): [ssl_version: ], [ssl_cipher: ]
INFO Beginning sync for InnoDB table gogs.repository
INFO Stream repository is using full table replication
INFO Table 'repository' exists
INFO Detected auto-incrementing primary key(s) - will replicate incrementally
INFO Running SELECT `external_wiki_url`,`external_tracker_url`,`num_pulls`,`pulls_ignore_whitespace`,`website`,`size`,`enable_external_wiki`,`updated_unix`,`use_custom_avatar`,`is_private`,`external_tracker_style`,`allow_public_issues`,`num_watches`,`description`,`default_branch`,`allow_public_wiki`,`num_milestones`,`num_closed_milestones`,`enable_external_tracker`,`fork_id`,`owner_id`,`is_fork`,`num_issues`,`is_mirror`,`id`,`num_closed_issues`,`name`,`external_tracker_format`,`enable_issues`,`num_stars`,`pulls_allow_rebase`,`lower_name`,`num_closed_pulls`,`enable_pulls`,`is_bare`,`num_forks`,`created_unix`,`enable_wiki` FROM `gogs`.`repository` WHERE `id` <= 1 ORDER BY `id` ASC
/Users/dalong/mylearning/python-virtualenv/mysql/lib/python3.7/site-packages/pymysql/connections.py:1077: UserWarning: Previous unbuffered result was left incomplete
warnings.warn("Previous unbuffered result was left incomplete")
INFO METRIC: {"type": "counter", "metric": "record_count", "value": 1, "tags": {"database": "gogs", "table": "repository"}}
INFO METRIC: {"type": "timer", "metric": "job_duration", "value": 0.054605960845947266, "tags": {"job_type": "sync_table", "database": "gogs", "table": "repository", "status": "succeeded"}}
INFO Loading 1 rows into 'repository'
INFO COPY repository_temp ("allow_public_issues", "allow_public_wiki", "created_unix", "default_branch", "description", "enable_external_tracker", "enable_external_wiki", "enable_issues", "enable_pulls", "enable_wiki", "external_tracker_format", "external_tracker_style", "external_tracker_url", "external_wiki_url", "fork_id", "id", "is_bare", "is_fork", "is_mirror", "is_private", "lower_name", "name", "num_closed_issues", "num_closed_milestones", "num_closed_pulls", "num_forks", "num_issues", "num_milestones", "num_pulls", "num_stars", "num_watches", "owner_id", "pulls_allow_rebase", "pulls_ignore_whitespace", "size", "updated_unix", "use_custom_avatar", "website") FROM STDIN WITH (FORMAT CSV, ESCAPE '\')
INFO UPDATE 1
INFO INSERT 0 0
{"currently_syncing": null, "bookmarks": {"gogs-repository": {"initial_full_table_complete": true}}}
-
create access_token from gitlab app
-
create gitlab virtualenv
virtualenv gitlab
source ./gitlab/bin/activate
pip install tap-gitlab
- add gitlab tap config
{
"api_url": "https://gitlab.com/api/v4",
"private_token": "<token>",
"groups": "<yougroup>",
"projects": "<you project>",
"start_date":"2010-01-01T00:00:00Z"
}
- running
./gitlab/bin/tap-gitlab -c gitlab.json | ./postgres/bin/target-postgres -c target.json
- config propeerties
{
"streams": [
{
"table_name": "loginusers",
"stream": "loginusers",
"metadata": [
{
"breadcrumb": [],
"metadata": {
"database-name": "usersapp",
"row-count": 3,
+ "selected": true,
+ "replication-method": "FULL_TABLE",
+ "custom-select-clause": "_id,name,age"
}
}
],
"tap_stream_id": "usersapp-loginusers",
"schema": {
"type": "object",
+ "properties": {
+ "name": {
+ "inclusion": "available",
+ "maxLength": 255,
+ "type": [
+ "null",
+ "string"
+ ]
+ },
+ "age": {
+ "inclusion": "available",
+ "maxLength": 255,
+ "type": [
+ "null",
+ "number"
+ ]
+ },
+ "type": {
+ "inclusion": "available",
+ "maxLength": 255,
+ "type": [
+ "null",
+ "string"
+ ]
+ },
+ "_id": {
+ "inclusion": "available",
+ "maxLength": 255,
+ "type": [
+ "null",
+ "string"
+ ]
+ }
+ }
+ }
+ }
+ ]
}
- run
./mongodb/bin/tap-mongodb -c mongo.json --properties usersapp.json | ./postgres/bin/target-po
stgres -c target.json