Create & Init Project...

app/job/main/upload/BUILD

@@ -0,0 +1,20 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/upload/cmd:all-srcs",
+        "//app/job/main/upload/conf:all-srcs",
+        "//app/job/main/upload/dao:all-srcs",
+        "//app/job/main/upload/http:all-srcs",
+        "//app/job/main/upload/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/upload/CHANGELOG.md

@@ -0,0 +1,18 @@
+# v1.2.2
+> 1.记录行为日志提交失败
+
+# v1.2.1
+> 1.增加打分 log
+
+# v1.2.0
+> 1.打分记录保存到 db
+
+# v1.1.0
+> 1.记录 ai 打分  
+> 2.接入行为日志
+
+# v1.0.0
+> 1.初始化项目  
+> 2.修复配置文件  
+> 3.优化消息格式  
+> 4.异步生产消息到ai 

app/job/main/upload/CONTRIBUTORS.md

@@ -0,0 +1,12 @@
+# Owner
+zhapuyu
+renwei
+zhuangzhewei
+liangkai
+
+# Author
+zhuangzhewei
+
+# Reviewer
+zhapuyu
+liangkai

app/job/main/upload/OWNERS

@@ -0,0 +1,17 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- liangkai
+- renwei
+- zhapuyu
+- zhuangzhewei
+labels:
+- job
+- job/main/upload
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- liangkai
+- zhapuyu
+- zhuangzhewei

app/job/main/upload/README.md

@@ -0,0 +1,12 @@
+# yellowing
+
+# 项目简介
+> bfs 鉴黄服务后台 job
+
+# 编译环境
+> 请只用golang v1.8.x以上版本编译执行。
+
+# 依赖包
+> 1.公共包go-common
+
+	

app/job/main/upload/cmd/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["bfs-upload-job.toml"],
+    importpath = "go-common/app/job/main/upload/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/upload/conf:go_default_library",
+        "//app/job/main/upload/http:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus/report:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/upload/cmd/bfs-upload-job.toml

@@ -0,0 +1,150 @@
+# This is a TOML document. Boom
+
+version = "1.0.0"
+user = "nobody"
+pid = "/tmp/bfs-upload-job.pid"
+dir = "./"
+DonwloadHost = "http://uat-i0.hdslb.com"
+UploadHost = "http://uat-bfs.bilibili.co"
+UploadAdminHost = "http://uat-manager.bilibili.co/x/admin/upload"
+HTTPTimeout = "5s"
+
+[bm]
+	[bm.inner]
+	addr = "0.0.0.0:9001"
+	maxListen = 10
+	timeout = "1s"
+	[bm.local]
+	addr = "0.0.0.0:9002"
+	maxListen = 10
+	timeout = "1s"
+
+[multiHTTP]
+	[multiHTTP.outer]
+	addrs = ["0.0.0.0:6121"]
+	maxListen = 10
+	[multiHTTP.inner]
+	addrs = ["0.0.0.0:6122"]
+	maxListen = 10
+	[multiHTTP.local]
+	addrs = ["0.0.0.0:6123"]
+	maxListen = 10
+
+[identify]
+	whiteAccessKey = ""
+	whiteMid = 0
+	[identify.app]
+	key = "6a29f8ed87407c11"
+	secret = "d3c5a85f5b895a03735b5d20a273bc57"
+	[identify.memcache]
+	name = "go-business/identify"
+	proto = "tcp"
+	addr = "172.16.33.54:11211"
+	active = 5
+	idle = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "80s"
+	[identify.host]
+	auth = "http://uat-passport.bilibili.com"
+	secret = "http://uat-open.bilibili.com"
+	[identify.httpClient]
+	key = "f022126a8a365e20"
+	secret = "b7b86838145d634b487e67b811b8fab2"
+	dial = "30ms"
+	timeout = "100ms"
+	keepAlive = "60s"
+	[identify.httpClient.breaker]
+	window  = "10s"
+	sleep   = "100ms"
+	bucket  = 10
+	ratio   = 0.5
+	request = 100
+	[identify.httpClient.url]
+	"http://uat-passport.bilibili.co/intranet/auth/tokenInfo" = {timeout = "100ms"}
+	"http://uat-passport.bilibili.co/intranet/auth/cookieInfo" = {timeout = "100ms"}
+	"http://uat-open.bilibili.co/api/getsecret" = {timeout = "500ms"}
+
+[ecode]
+	domain = "uat-api.bilibili.co"
+	all = "1h"
+	diff = "5m"
+	[ecode.clientconfig]
+		key = "c1a1cb2d89c33794"
+		secret = "dda47eeca111e03e6845017505baea13"
+		dial      = "2000ms"
+		timeout   = "2s"
+		keepAlive = "10s"
+		timer     = 128
+		[ecode.clientconfig.breaker]
+			window  ="3s"
+			sleep   ="100ms"
+			bucket  = 10
+			ratio   = 0.5
+			request = 100
+
+[Databus]
+	key = "dbe67e6a4c36f877"
+	secret = "8c775ea242caa367ba5c876c04576571"
+	group= "Bfs-SHY-MainCommonArch-S"
+	topic= "Bfs-SHY-T"
+	action="sub"
+	proto = "tcp"
+	addr = "172.18.33.50:6205"
+	idle = 100
+	active = 100
+	dialTimeout = "1s"
+	readTimeout = "60s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+
+[AIYellowing]
+	ExceptBuckets = []
+
+	[AIYellowing.producer]
+	key = "dbe67e6a4c36f877"
+	secret = "8c775ea242caa367ba5c876c04576571"
+	group= "Bfs-Yellow-Req-MainCommonArch-P"
+	topic= "Bfs-Yellow-Req-T"
+	action="pub"
+	proto = "tcp"
+	addr = "172.18.33.50:6205"
+	idle = 100
+	active = 100
+	dialTimeout = "1s"
+	readTimeout = "60s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+
+	[AIYellowing.consumer]
+	key = "dbe67e6a4c36f877"
+	secret = "8c775ea242caa367ba5c876c04576571"
+	group= "Bfs-Yellow-Res-MainCommonArch-S"
+	topic= "Bfs-Yellow-Res-T"
+	action="sub"
+	proto = "tcp"
+	addr = "172.18.33.50:6205"
+	idle = 100
+	active = 100
+	dialTimeout = "1s"
+	readTimeout = "60s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+
+[threshold]
+	sex = 9500
+
+[httpClient]
+	key = "ad4bb9b8f5d9d4a7"
+    secret = "6912080d78d58be7cb94f57d50d438f6"
+    dial = "1s"
+    timeout = "10s"
+    keepAlive = "60s"
+    timer = 1000
+    [httpClient.breaker]
+        window  = "10s"
+        sleep   = "100ms"
+        bucket  = 10
+        ratio   = 0.5
+        request = 100

app/job/main/upload/cmd/main.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/main/upload/conf"
+	"go-common/app/job/main/upload/http"
+	"go-common/library/log"
+	"go-common/library/queue/databus/report"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	// init log
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("upload start")
+	// service init
+	http.Init(conf.Conf)
+	// init pprof conf.Conf.Perf
+	// report
+	report.InitManager(nil)
+	// init signal
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("upload get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("upload exit")
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/main/upload/conf/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/upload/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/upload/conf/conf.go

@@ -0,0 +1,120 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/conf"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+// global var
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config config set
+type Config struct {
+	// base
+	// elk
+	Log *log.Config
+	// http
+	BM *HTTPServers
+	// tracer
+	Tracer *trace.Config
+	// ecode
+	Ecode *ecode.Config
+
+	DonwloadHost    string
+	UploadHost      string
+	UploadAdminHost string
+	RoutineCount    int
+	HTTPClient      *bm.ClientConfig // call upload admin
+	HTTPTimeout     xtime.Duration
+	Databus         *databus.Config
+	AIYellowing     AIYellowing
+	Threshold       *Threshold
+}
+
+// AIYellowing describe AI upload config
+type AIYellowing struct {
+	ExceptBuckets []string
+	Producer      *databus.Config
+	Consumer      *databus.Config
+}
+
+// Threshold .
+type Threshold struct {
+	Sex      int
+	Politics int
+	Blood    int
+	Violent  int
+}
+
+// HTTPServers Http Servers
+type HTTPServers struct {
+	Outer *bm.ServerConfig
+	Inner *bm.ServerConfig
+	Local *bm.ServerConfig
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/upload/dao/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dao.go"],
+    importpath = "go-common/app/job/main/upload/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//app/job/main/upload/conf:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["dao_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/upload/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/upload/dao/dao.go

@@ -0,0 +1,30 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/main/upload/conf"
+)
+
+// Dao dao
+type Dao struct {
+	c *conf.Config
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c: c,
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	return nil
+}

app/job/main/upload/dao/dao_test.go

@@ -0,0 +1,35 @@
+package dao
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/main/upload/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var d *Dao
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/bfs-upload-job.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	d = New(conf.Conf)
+}
+
+func TestPing(t *testing.T) {
+	Convey("Ping", t, func() {
+		err := d.Ping(context.TODO())
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestClose(t *testing.T) {
+	Convey("Ping", t, func() {
+		err := d.Close
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/upload/http/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/upload/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/upload/conf:go_default_library",
+        "//app/job/main/upload/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/upload/http/http.go

@@ -0,0 +1,58 @@
+package http
+
+import (
+	"go-common/app/job/main/upload/conf"
+	"go-common/app/job/main/upload/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	svc *service.Service
+)
+
+// Init init
+func Init(c *conf.Config) {
+	initService(c)
+	// init router
+	engineInner := bm.DefaultServer(c.BM.Inner)
+	outerRouter(engineInner)
+	if err := engineInner.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+	engineLocal := bm.DefaultServer(c.BM.Local)
+	localRouter(engineLocal)
+	if err := engineLocal.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+// initService init services.
+func initService(c *conf.Config) {
+	svc = service.New(c)
+}
+
+// outerRouter init outer router api path.
+func outerRouter(e *bm.Engine) {
+	//init api
+	e.GET("/monitor/ping", ping)
+	group := e.Group("/x/job/main/upload")
+	{
+		group.GET("")
+	}
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	svc.Ping(c)
+}
+
+// innerRouter init local router api path.
+func localRouter(e *bm.Engine) {
+	group := e.Group("")
+	{
+		group.GET("")
+	}
+}

app/job/main/upload/service/BUILD

@@ -0,0 +1,55 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "aiyellowing.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/main/upload/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/upload/conf:go_default_library",
+        "//app/job/main/upload/dao:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/queue/databus/report:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "bucket_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/upload/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/upload/service/aiyellowing.go

@@ -0,0 +1,96 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"time"
+
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/queue/databus/report"
+)
+
+// aiReq is a producer send file data to eroticism inspect serve
+func (j *Job) aiReq(ctx context.Context, meta *Meta) {
+	url := fmt.Sprintf(_downloadFmt, j.downloadHost, meta.Bucket, meta.Filename)
+	msg := &AIReqMessage{URL: url, FileName: meta.Filename, Bucket: meta.Bucket}
+	if err := j.AIYellowingProducer.Send(ctx, meta.Filename, msg); err != nil {
+		log.Error("d.databus.Send(%s,%v) error(%v)", meta.Bucket, *msg, err)
+		return
+	}
+	log.Info("send msg to ai success: key:(%s),value:(%+v)", meta.Filename, msg)
+}
+
+// aiResp is a consumer receive message from eroticism inspect serve
+func (j *Job) aiResp(ctx context.Context) {
+	var (
+		msgs  <-chan *databus.Message
+		ok    bool
+		msg   *databus.Message
+		aiMsg *AIRespMessage
+		err   error
+	)
+	msgs = j.AIYellowingConsumer.Messages() //ai response message
+	for {
+		if msg, ok = <-msgs; !ok {
+			log.Info("databus Consumer exit")
+			break
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit() error(%v)", err)
+			continue
+
+		}
+		aiMsg = new(AIRespMessage)
+		// todo: parse kafka message from ai and judge if add to db
+		if err = json.Unmarshal(msg.Value, aiMsg); err != nil {
+			log.Error("Job.run.Unmarshal(key:%s, value:%s),err(%v)", msg.Key, string(msg.Value), err)
+			continue
+		}
+		log.Info("consume msg from ai success: key:(%s),value:(%+v)", msg.Key, aiMsg)
+		if aiMsg.ErrorCode != 0 {
+			log.Error("ai audit failed code: %d, message: %s", aiMsg.ErrorCode, aiMsg.ErrorMsg)
+			continue
+		}
+
+		// deal ai result
+		// sex > threshold will save in db
+		sex := int(math.Round(aiMsg.Sex * 10000))
+		politics := int(math.Round(aiMsg.Politics * 10000))
+		blood := int(math.Round(aiMsg.Blood * 10000))
+		violent := int(math.Round(aiMsg.Violent * 10000))
+		if sex > j.Threshold.Sex || politics > j.Threshold.Politics || blood > j.Threshold.Blood || violent > j.Threshold.Violent {
+			log.Warn("url:%s need save db points(sex:%d,politics:%d,blood:%d,violent:%d)", aiMsg.URL, sex, politics, blood, violent)
+			if err = j.RetryAddRecord(aiMsg); err != nil {
+				log.Error("j.RetryAddRecord(%+v) failed(%v)", aiMsg, err)
+				continue
+			} else {
+				log.Info("call upload-admin success: %+v", aiMsg)
+			}
+		}
+		//todo: send to audit log platform
+		m := &report.ManagerInfo{
+			Uname:    "ai",
+			UID:      0,
+			Business: 161, //bfs 上传审核
+			Type:     1,   //ai审核
+			Action:   "ai",
+			Index:    []interface{}{aiMsg.URL, aiMsg.Bucket, aiMsg.FileName, int(math.Round(aiMsg.Sex * 10000)), int(math.Round(aiMsg.Politics * 10000))},
+			Ctime:    time.Now(),
+			Content: map[string]interface{}{ //分数
+				"sex":       aiMsg.Sex,
+				"violent":   aiMsg.Violent,
+				"politics":  aiMsg.Politics,
+				"blood":     aiMsg.Blood,
+				"is_yellow": aiMsg.IsYellow,
+			},
+		}
+		if err = report.Manager(m); err != nil {
+			log.Error("report.Manager() m(%+v) error:%v", m, err)
+			continue
+		}
+		log.Info("send log report success: (%+v) \n", m)
+	}
+}

app/job/main/upload/service/bucket_test.go

@@ -0,0 +1,19 @@
+package service
+
+import "testing"
+
+func TestGet(t *testing.T) {
+
+}
+
+func TestInitBuckets(t *testing.T) {
+
+}
+
+func TestString(t *testing.T) {
+
+}
+
+func TestPublic(t *testing.T) {
+
+}

app/job/main/upload/service/service.go

@@ -0,0 +1,235 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/url"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/upload/conf"
+	"go-common/app/job/main/upload/dao"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	xhttp "go-common/library/net/http/blademaster"
+	"go-common/library/queue/databus"
+)
+
+const (
+	_downloadFmt       = "%s/bfs/%s/%s"
+	_uploadAdminAddFmt = "%s/add"
+)
+
+// Service struct
+type Service struct {
+	c   *conf.Config
+	dao *dao.Dao
+}
+
+// Meta describe databus message value format from bfs proxy
+type Meta struct {
+	Bucket   string `json:"bucket"`
+	Filename string `json:"filename"`
+	Mine     string `json:"mine"`
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:   c,
+		dao: dao.New(c),
+	}
+	Run(c)
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}
+
+// Job .
+type Job struct {
+	downloadHost             string
+	uploadHost               string
+	uploadAdminHost          string
+	xclient                  *xhttp.Client
+	sub                      *databus.Databus
+	ch                       chan *Meta
+	routineCount             int
+	AIYellowingConsumer      *databus.Databus //consume response from ai
+	AIYellowingProducer      *databus.Databus //produce request to ai
+	AIYellowingExceptBuckets map[string]bool
+	Threshold                *conf.Threshold //score threshold
+}
+
+// AIReqMessage defined send and receive databus message format with AI
+type AIReqMessage struct {
+	Bucket   string `json:"bucket"`
+	FileName string `json:"file_name"`
+	URL      string `json:"url"`
+	IsYellow bool   `json:"is_yellow"`
+}
+
+//AIRespMessage defined response databus message format from AI
+type AIRespMessage struct {
+	URL       string  `json:"url"`
+	FileName  string  `json:"file_name"`
+	Bucket    string  `json:"bucket"`
+	Sex       float64 `json:"sex"`
+	Violent   float64 `json:"violent"`
+	Blood     float64 `json:"blood"`
+	Politics  float64 `json:"politics"`
+	IsYellow  bool    `json:"is_yellow"`
+	ErrorCode int64   `json:"error_code"`
+	ErrorMsg  string  `json:"error_msg"`
+}
+
+// Add describe params of /x/admin/bfs-upload/add
+type Add struct {
+	Bucket   string `json:"bucket"`
+	FileName string `json:"filename"`
+}
+
+// AddResp describe response of /x/admin/bfs-upload/add
+type AddResp struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+	TTL     int    `json:"ttl"`
+}
+
+// Run .
+func Run(cfg *conf.Config) {
+	exceptBuckets := make(map[string]bool)
+	for _, bucket := range cfg.AIYellowing.ExceptBuckets {
+		exceptBuckets[bucket] = false
+	}
+	if cfg.Threshold == nil {
+		cfg.Threshold = &conf.Threshold{
+			Sex:      5000,
+			Politics: 5000,
+			Blood:    5000,
+			Violent:  5000,
+		}
+	}
+	if cfg.Threshold.Sex == 0 {
+		cfg.Threshold.Sex = 5000
+	}
+	if cfg.Threshold.Politics == 0 {
+		cfg.Threshold.Politics = 5000
+	}
+	if cfg.Threshold.Blood == 0 {
+		cfg.Threshold.Blood = 5000
+	}
+	if cfg.Threshold.Violent == 0 {
+		cfg.Threshold.Violent = 5000
+	}
+
+	j := &Job{
+		downloadHost:             cfg.DonwloadHost,
+		uploadHost:               cfg.UploadHost,
+		uploadAdminHost:          cfg.UploadAdminHost,
+		routineCount:             cfg.RoutineCount,
+		xclient:                  bm.NewClient(cfg.HTTPClient),
+		sub:                      databus.New(cfg.Databus),
+		ch:                       make(chan *Meta, 1024),
+		AIYellowingConsumer:      databus.New(cfg.AIYellowing.Consumer),
+		AIYellowingProducer:      databus.New(cfg.AIYellowing.Producer),
+		AIYellowingExceptBuckets: exceptBuckets,
+		Threshold:                cfg.Threshold,
+	}
+	go j.aireqproc()
+	go j.proxysyncproc(context.Background())
+	go j.aiResp(context.Background()) // deal ai response
+}
+
+func (j *Job) aireqproc() {
+	for i := 0; i < 10; i++ {
+		go func() {
+			for meta := range j.ch {
+				j.aiReq(context.Background(), meta)
+			}
+		}()
+	}
+}
+
+// Run .
+func (j *Job) proxysyncproc(ctx context.Context) {
+	var (
+		err  error
+		meta *Meta
+	)
+	for {
+		msg, ok := <-j.sub.Messages()
+		if !ok {
+			log.Error("consume msg error, uploadproc exit!")
+			return
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit() error(%v)", err)
+			continue
+		}
+		meta = new(Meta)
+		if err = json.Unmarshal(msg.Value, meta); err != nil {
+			log.Error("Job.run.Unmarshal(key:%v),err(%v)", msg.Key, err)
+			continue
+		}
+		// if bucket need yellow inspects
+		_, ok = j.AIYellowingExceptBuckets[meta.Bucket]
+		if !ok {
+			log.Info("bfs-proxy sync a msg:(%+v) meta(%+v) need to req ai", msg, meta)
+			j.add(meta)
+		}
+	}
+}
+
+func (j *Job) add(item *Meta) {
+	j.ch <- item
+}
+
+// addRecord add a record into upload-admin if a pic is yellow
+func (j *Job) addRecord(aiMsg *AIRespMessage) (err error) {
+	var (
+		uploadAdminAddURL string
+	)
+
+	addResp := new(AddResp)
+	params := url.Values{}
+	params.Add("bucket", aiMsg.Bucket)
+	params.Add("filename", aiMsg.FileName)
+	params.Add("url", aiMsg.URL)
+	params.Add("sex", strconv.Itoa(int(math.Round(aiMsg.Sex*10000))))
+	params.Add("politics", strconv.Itoa(int(math.Round(aiMsg.Politics*10000))))
+
+	uploadAdminAddURL = fmt.Sprintf(_uploadAdminAddFmt, j.uploadAdminHost)
+
+	if err = j.xclient.Post(context.TODO(), uploadAdminAddURL, "", params, addResp); err != nil {
+		return
+	}
+	if addResp.Code != 0 {
+		log.Error("call /x/admin/bfs-upload/add code error, code(%d),message(%s)", addResp.Code, addResp.Message)
+		return
+	}
+	log.Info("upload-admin add success(%+v)", aiMsg)
+	return
+}
+
+// RetryAddRecord try to add a record to upload-admin
+func (j *Job) RetryAddRecord(aiMsg *AIRespMessage) (err error) {
+	attempts := 3
+	for i := 0; i < attempts; i++ {
+		err = j.addRecord(aiMsg)
+		if err == nil {
+			return
+		}
+		time.Sleep(1 * time.Second)
+	}
+	return
+}

app/job/main/upload/service/service_test.go

@@ -0,0 +1,48 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/upload/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/bfs-upload-job.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+	time.Sleep(time.Second)
+}
+
+func TestPing(t *testing.T) {
+	Convey("Ping", t, func() {
+		err := s.Ping(context.TODO())
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestRetryAddRecord(t *testing.T) {
+
+}
+
+func TestRun(t *testing.T) {
+	Convey("Run", t, func() {
+		Run(s.c)
+	})
+}
+
+func TestClose(t *testing.T) {
+	Convey("Close", t, func() {
+		s.Close()
+	})
+}