Create & Init Project...

app/job/main/upload/service/BUILD

@@ -0,0 +1,55 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "aiyellowing.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/main/upload/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/upload/conf:go_default_library",
+        "//app/job/main/upload/dao:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/queue/databus/report:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "bucket_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/upload/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/upload/service/aiyellowing.go

@@ -0,0 +1,96 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"time"
+
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/queue/databus/report"
+)
+
+// aiReq is a producer send file data to eroticism inspect serve
+func (j *Job) aiReq(ctx context.Context, meta *Meta) {
+	url := fmt.Sprintf(_downloadFmt, j.downloadHost, meta.Bucket, meta.Filename)
+	msg := &AIReqMessage{URL: url, FileName: meta.Filename, Bucket: meta.Bucket}
+	if err := j.AIYellowingProducer.Send(ctx, meta.Filename, msg); err != nil {
+		log.Error("d.databus.Send(%s,%v) error(%v)", meta.Bucket, *msg, err)
+		return
+	}
+	log.Info("send msg to ai success: key:(%s),value:(%+v)", meta.Filename, msg)
+}
+
+// aiResp is a consumer receive message from eroticism inspect serve
+func (j *Job) aiResp(ctx context.Context) {
+	var (
+		msgs  <-chan *databus.Message
+		ok    bool
+		msg   *databus.Message
+		aiMsg *AIRespMessage
+		err   error
+	)
+	msgs = j.AIYellowingConsumer.Messages() //ai response message
+	for {
+		if msg, ok = <-msgs; !ok {
+			log.Info("databus Consumer exit")
+			break
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit() error(%v)", err)
+			continue
+
+		}
+		aiMsg = new(AIRespMessage)
+		// todo: parse kafka message from ai and judge if add to db
+		if err = json.Unmarshal(msg.Value, aiMsg); err != nil {
+			log.Error("Job.run.Unmarshal(key:%s, value:%s),err(%v)", msg.Key, string(msg.Value), err)
+			continue
+		}
+		log.Info("consume msg from ai success: key:(%s),value:(%+v)", msg.Key, aiMsg)
+		if aiMsg.ErrorCode != 0 {
+			log.Error("ai audit failed code: %d, message: %s", aiMsg.ErrorCode, aiMsg.ErrorMsg)
+			continue
+		}
+
+		// deal ai result
+		// sex > threshold will save in db
+		sex := int(math.Round(aiMsg.Sex * 10000))
+		politics := int(math.Round(aiMsg.Politics * 10000))
+		blood := int(math.Round(aiMsg.Blood * 10000))
+		violent := int(math.Round(aiMsg.Violent * 10000))
+		if sex > j.Threshold.Sex || politics > j.Threshold.Politics || blood > j.Threshold.Blood || violent > j.Threshold.Violent {
+			log.Warn("url:%s need save db points(sex:%d,politics:%d,blood:%d,violent:%d)", aiMsg.URL, sex, politics, blood, violent)
+			if err = j.RetryAddRecord(aiMsg); err != nil {
+				log.Error("j.RetryAddRecord(%+v) failed(%v)", aiMsg, err)
+				continue
+			} else {
+				log.Info("call upload-admin success: %+v", aiMsg)
+			}
+		}
+		//todo: send to audit log platform
+		m := &report.ManagerInfo{
+			Uname:    "ai",
+			UID:      0,
+			Business: 161, //bfs 上传审核
+			Type:     1,   //ai审核
+			Action:   "ai",
+			Index:    []interface{}{aiMsg.URL, aiMsg.Bucket, aiMsg.FileName, int(math.Round(aiMsg.Sex * 10000)), int(math.Round(aiMsg.Politics * 10000))},
+			Ctime:    time.Now(),
+			Content: map[string]interface{}{ //分数
+				"sex":       aiMsg.Sex,
+				"violent":   aiMsg.Violent,
+				"politics":  aiMsg.Politics,
+				"blood":     aiMsg.Blood,
+				"is_yellow": aiMsg.IsYellow,
+			},
+		}
+		if err = report.Manager(m); err != nil {
+			log.Error("report.Manager() m(%+v) error:%v", m, err)
+			continue
+		}
+		log.Info("send log report success: (%+v) \n", m)
+	}
+}

app/job/main/upload/service/bucket_test.go

@@ -0,0 +1,19 @@
+package service
+
+import "testing"
+
+func TestGet(t *testing.T) {
+
+}
+
+func TestInitBuckets(t *testing.T) {
+
+}
+
+func TestString(t *testing.T) {
+
+}
+
+func TestPublic(t *testing.T) {
+
+}

app/job/main/upload/service/service.go

@@ -0,0 +1,235 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/url"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/upload/conf"
+	"go-common/app/job/main/upload/dao"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	xhttp "go-common/library/net/http/blademaster"
+	"go-common/library/queue/databus"
+)
+
+const (
+	_downloadFmt       = "%s/bfs/%s/%s"
+	_uploadAdminAddFmt = "%s/add"
+)
+
+// Service struct
+type Service struct {
+	c   *conf.Config
+	dao *dao.Dao
+}
+
+// Meta describe databus message value format from bfs proxy
+type Meta struct {
+	Bucket   string `json:"bucket"`
+	Filename string `json:"filename"`
+	Mine     string `json:"mine"`
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:   c,
+		dao: dao.New(c),
+	}
+	Run(c)
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}
+
+// Job .
+type Job struct {
+	downloadHost             string
+	uploadHost               string
+	uploadAdminHost          string
+	xclient                  *xhttp.Client
+	sub                      *databus.Databus
+	ch                       chan *Meta
+	routineCount             int
+	AIYellowingConsumer      *databus.Databus //consume response from ai
+	AIYellowingProducer      *databus.Databus //produce request to ai
+	AIYellowingExceptBuckets map[string]bool
+	Threshold                *conf.Threshold //score threshold
+}
+
+// AIReqMessage defined send and receive databus message format with AI
+type AIReqMessage struct {
+	Bucket   string `json:"bucket"`
+	FileName string `json:"file_name"`
+	URL      string `json:"url"`
+	IsYellow bool   `json:"is_yellow"`
+}
+
+//AIRespMessage defined response databus message format from AI
+type AIRespMessage struct {
+	URL       string  `json:"url"`
+	FileName  string  `json:"file_name"`
+	Bucket    string  `json:"bucket"`
+	Sex       float64 `json:"sex"`
+	Violent   float64 `json:"violent"`
+	Blood     float64 `json:"blood"`
+	Politics  float64 `json:"politics"`
+	IsYellow  bool    `json:"is_yellow"`
+	ErrorCode int64   `json:"error_code"`
+	ErrorMsg  string  `json:"error_msg"`
+}
+
+// Add describe params of /x/admin/bfs-upload/add
+type Add struct {
+	Bucket   string `json:"bucket"`
+	FileName string `json:"filename"`
+}
+
+// AddResp describe response of /x/admin/bfs-upload/add
+type AddResp struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+	TTL     int    `json:"ttl"`
+}
+
+// Run .
+func Run(cfg *conf.Config) {
+	exceptBuckets := make(map[string]bool)
+	for _, bucket := range cfg.AIYellowing.ExceptBuckets {
+		exceptBuckets[bucket] = false
+	}
+	if cfg.Threshold == nil {
+		cfg.Threshold = &conf.Threshold{
+			Sex:      5000,
+			Politics: 5000,
+			Blood:    5000,
+			Violent:  5000,
+		}
+	}
+	if cfg.Threshold.Sex == 0 {
+		cfg.Threshold.Sex = 5000
+	}
+	if cfg.Threshold.Politics == 0 {
+		cfg.Threshold.Politics = 5000
+	}
+	if cfg.Threshold.Blood == 0 {
+		cfg.Threshold.Blood = 5000
+	}
+	if cfg.Threshold.Violent == 0 {
+		cfg.Threshold.Violent = 5000
+	}
+
+	j := &Job{
+		downloadHost:             cfg.DonwloadHost,
+		uploadHost:               cfg.UploadHost,
+		uploadAdminHost:          cfg.UploadAdminHost,
+		routineCount:             cfg.RoutineCount,
+		xclient:                  bm.NewClient(cfg.HTTPClient),
+		sub:                      databus.New(cfg.Databus),
+		ch:                       make(chan *Meta, 1024),
+		AIYellowingConsumer:      databus.New(cfg.AIYellowing.Consumer),
+		AIYellowingProducer:      databus.New(cfg.AIYellowing.Producer),
+		AIYellowingExceptBuckets: exceptBuckets,
+		Threshold:                cfg.Threshold,
+	}
+	go j.aireqproc()
+	go j.proxysyncproc(context.Background())
+	go j.aiResp(context.Background()) // deal ai response
+}
+
+func (j *Job) aireqproc() {
+	for i := 0; i < 10; i++ {
+		go func() {
+			for meta := range j.ch {
+				j.aiReq(context.Background(), meta)
+			}
+		}()
+	}
+}
+
+// Run .
+func (j *Job) proxysyncproc(ctx context.Context) {
+	var (
+		err  error
+		meta *Meta
+	)
+	for {
+		msg, ok := <-j.sub.Messages()
+		if !ok {
+			log.Error("consume msg error, uploadproc exit!")
+			return
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit() error(%v)", err)
+			continue
+		}
+		meta = new(Meta)
+		if err = json.Unmarshal(msg.Value, meta); err != nil {
+			log.Error("Job.run.Unmarshal(key:%v),err(%v)", msg.Key, err)
+			continue
+		}
+		// if bucket need yellow inspects
+		_, ok = j.AIYellowingExceptBuckets[meta.Bucket]
+		if !ok {
+			log.Info("bfs-proxy sync a msg:(%+v) meta(%+v) need to req ai", msg, meta)
+			j.add(meta)
+		}
+	}
+}
+
+func (j *Job) add(item *Meta) {
+	j.ch <- item
+}
+
+// addRecord add a record into upload-admin if a pic is yellow
+func (j *Job) addRecord(aiMsg *AIRespMessage) (err error) {
+	var (
+		uploadAdminAddURL string
+	)
+
+	addResp := new(AddResp)
+	params := url.Values{}
+	params.Add("bucket", aiMsg.Bucket)
+	params.Add("filename", aiMsg.FileName)
+	params.Add("url", aiMsg.URL)
+	params.Add("sex", strconv.Itoa(int(math.Round(aiMsg.Sex*10000))))
+	params.Add("politics", strconv.Itoa(int(math.Round(aiMsg.Politics*10000))))
+
+	uploadAdminAddURL = fmt.Sprintf(_uploadAdminAddFmt, j.uploadAdminHost)
+
+	if err = j.xclient.Post(context.TODO(), uploadAdminAddURL, "", params, addResp); err != nil {
+		return
+	}
+	if addResp.Code != 0 {
+		log.Error("call /x/admin/bfs-upload/add code error, code(%d),message(%s)", addResp.Code, addResp.Message)
+		return
+	}
+	log.Info("upload-admin add success(%+v)", aiMsg)
+	return
+}
+
+// RetryAddRecord try to add a record to upload-admin
+func (j *Job) RetryAddRecord(aiMsg *AIRespMessage) (err error) {
+	attempts := 3
+	for i := 0; i < attempts; i++ {
+		err = j.addRecord(aiMsg)
+		if err == nil {
+			return
+		}
+		time.Sleep(1 * time.Second)
+	}
+	return
+}

app/job/main/upload/service/service_test.go

@@ -0,0 +1,48 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/upload/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/bfs-upload-job.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+	time.Sleep(time.Second)
+}
+
+func TestPing(t *testing.T) {
+	Convey("Ping", t, func() {
+		err := s.Ping(context.TODO())
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestRetryAddRecord(t *testing.T) {
+
+}
+
+func TestRun(t *testing.T) {
+	Convey("Run", t, func() {
+		Run(s.c)
+	})
+}
+
+func TestClose(t *testing.T) {
+	Convey("Close", t, func() {
+		s.Close()
+	})
+}