Create & Init Project...

app/job/main/archive/cmd/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = [
+        "archive-job-example.toml",
+        "archive-job-test.toml",
+    ],
+    importpath = "go-common/app/job/main/archive/cmd",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//app/job/main/archive/http:go_default_library",
+        "//app/job/main/archive/service:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/cmd/archive-job-example.toml

@@ -0,0 +1,77 @@
+version = "1.0.0"
+user    = "nobody"
+pid     = "/tmp/archive-job.pid"
+dir     = "./"
+perf    = "127.0.0.1:6161"
+trace   = false
+debug   = false
+env     = "test"
+
+tick = "50ms"
+
+[xlog]
+dir = "/data/log/archive-job/"
+
+[app]
+key    = "e7482d29be4a95b8"
+secret = "9e803791cdef756e75faee68e12b7442"
+
+[host]
+apiCo = "http://api.bilibili.co"
+
+[httpClient]
+dial      = "100ms"
+timeout   = "1s"
+keepAlive = "60s"
+timer     = 128
+
+[databus]
+key = "0Pub71WwEMKXu63qtztu"
+secret = "0Pub71WwEMKXu63qtztv"
+group = "Stat-UGC-S"
+topic = "Stat-T"
+action = "sub"
+    [databus.redis]
+    name = "archive-job/databus"
+    proto = "tcp"
+    addr = "172.16.33.56:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[kafka]
+group = "archive-job-consumer"
+topics = ["archive-push", "stat"]
+offset = false
+    [kafka.zookeeper]
+    root = "/kafka"
+    addrs = ["172.16.0.148:2181"]
+    timeout = "30s"
+
+[archiveRPC]
+    pullInterval = "10s"
+    [archiveRPC.client]
+    proto = "tcp"
+    addr = "172.16.0.26:6089"
+    timeout = "1s"
+    timer = 1000
+    [[archiveRPC.backup]]
+    proto = "tcp"
+    addr = "172.16.0.26:6089"
+    timeout = "1s"
+    timer = 1000
+    [archiveRPC.zookeeper]
+    root = "/microservice/archive-service/"
+    addrs = ["172.16.0.148:2181"]
+    timeout = "30s"
+
+[mail]
+host = "smtp.exmail.qq.com"
+port = 465
+username = "2333@bilibili.com"
+password = "2333"
+bangumi = ["233@bilibili.com", "2333@bilibili.com"]
+movie = ["233@bilibili.com", "2333@bilibili.com"]

app/job/main/archive/cmd/archive-job-test.toml

@@ -0,0 +1,223 @@
+env = "uat"
+chanSize = 50
+PGCAsync = 0
+UGCAsync = 0
+MonitorSize = 30
+WeChatToken = "uYsRqXPjnbyXDCba"
+WeChatSecret = "tUILeGIXJHFSUdwfLjBHDZJXmLgYpgEy"
+WeChantUsers = "peiyifei,haoguanwei"
+tick = "50ms"
+
+[bm]
+addr = "0.0.0.0:6161"
+timeout = "1s"
+[app]
+key    = "53e2fa226f5ad348"
+secret = "3cf6bd1b0ff671021da5f424fea4b04a"
+
+[host]
+apiCo = "http://api.bilibili.co"
+
+[httpClient]
+key    = "53e2fa226f5ad348"
+secret = "3cf6bd1b0ff671021da5f424fea4b04a"
+dial      = "100ms"
+timeout   = "1s"
+keepAlive = "60s"
+[httpClient.breaker]
+    window  = "10s"
+    sleep   = "100ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+
+[db]
+    [db.archive]
+    name = "172.18.33.49:3306"
+    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_archive?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 20
+    idle = 10
+    idleTimeout = "4h"
+    queryTimeout = "5s"
+    execTimeout = "5s"
+    tranTimeout = "5s"
+    [db.archive.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+    [db.result]
+    name = "172.18.33.49:3306"
+    dsn = "test:test@tcp(172.16.33.205:3308)/archive_result?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8"
+    active = 20
+    idle = 10
+    idleTimeout = "4h"
+    queryTimeout = "5s"
+    execTimeout = "5s"
+    tranTimeout = "5s"
+    [db.result.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+    [db.dede]
+    name = "172.18.33.49:3307"
+    dsn = "dede:jh3GWwFgzsLHvkWsDPiSxy5Ge2vNt4Ys@tcp(172.18.33.49:3307)/bilibili?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8"
+    active = 20
+    idle = 10
+    idleTimeout ="4h"
+    queryTimeout = "5s"
+    execTimeout = "5s"
+    tranTimeout = "5s"
+    [db.dede.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[redis]
+name = "archive-job/trackreport"
+proto = "tcp"
+addr = "172.18.33.61:6807"
+active = 10
+idle = 1
+dialTimeout = "1s"
+readTimeout = "1s"
+writeTimeout = "1s"
+idleTimeout = "80s"
+
+[archiveResultPub]
+key = "0QNB0ZgFozbKUCQhbTq8"
+secret = "0QNB0ZgFozbKUCQhbTq9"
+group = "ArchiveNotify-ArchiveJob-P"
+topic = "ArchiveNotify-T"
+action = "pub"
+name = "archive-job/archivepub"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 10
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[statSub]
+key = "0Q2UxI6hM1i6k6LRzDmx"
+secret= "0Q2UxI6hM1i6k6LRzDmy"
+group= "Stat-UGC-S"
+topic= "Stat-T"
+action="sub"
+name = "archive-job/stat"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+active = 1
+idle = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+expire = "1h"
+
+[DmSub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "DMSubject-MainAppSvr-S"
+topic = "DMSubject-T"
+action = "sub"
+name = "archive-job/DMSubject"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[DmPub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatDM-MainAppSvr-P"
+topic = "StatDM-T"
+action = "pub"
+name = "archive-job/dmpub"
+proto = "tcp"
+addr = "172.16.33.158:6205"
+idle = 100
+active = 100
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[VideoupSub]
+key = "0QNB0ZgFozbKUCQhbTq8"
+secret = "0QNB0ZgFozbKUCQhbTq9"
+group = "ArchiveJob-Videoup2Bvc-S"
+topic = "Videoup2Bvc"
+action = "sub"
+name = "archive-job/videoupsub"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[CacheSub]
+key = "8e27ab7e39270b59"
+secret = "477df6a068d7332a163f95abbad2079c"
+group = "ArchiveUserCache-MainAppSvr-S"
+topic = "ArchiveUserCache-T"
+action = "sub"
+name = "archive-job/userCache"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "80s"
+
+[AccountNotifySub]
+key = "8e27ab7e39270b59"
+secret = "477df6a068d7332a163f95abbad2079c"
+group = "AccountNotify-MainAppSvr-S"
+topic = "AccountNotify-T"
+action = "sub"
+name = "archive-job/userCache"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "80s"
+[AccountNotifySub.discovery]
+    domain = "api.bilibili.co"
+    key = "53e2fa226f5ad348"
+    secret = "3cf6bd1b0ff671021da5f424fea4b04a"
+    region = "sh"
+    zone = "sh001"
+    env = "prod"
+
+[mail]
+host = "smtp.exmail.qq.com"
+port = 465
+username = "2333@bilibili.com"
+password = "2333"
+bangumi = ["233@bilibili.com"]
+movie = ["233@bilibili.com"]
+
+[[ArchiveServices]]
+cluster="group1"
+[[ArchiveServices]]
+cluster="group2"

app/job/main/archive/cmd/main.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/main/archive/conf"
+	"go-common/app/job/main/archive/http"
+	"go-common/app/job/main/archive/service"
+	"go-common/library/log"
+)
+
+var (
+	srv *service.Service
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	// init log
+	log.Init(conf.Conf.XLog)
+	defer log.Close()
+	log.Info("archive-job start")
+	srv = service.New(conf.Conf)
+	http.Init(conf.Conf, srv)
+	signalHandler()
+}
+
+func signalHandler() {
+	var (
+		err error
+		ch  = make(chan os.Signal, 1)
+	)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		si := <-ch
+		switch si {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("get a signal %s, stop the consume process", si.String())
+			if err = srv.Close(); err != nil {
+				log.Error("srv close consumer error(%v)", err)
+			}
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}