Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,38 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/live/recommend-job/internal/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/cache/redis:go_default_library",
"//library/conf:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/net/trace:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,108 @@
package conf
import (
"errors"
"flag"
"go-common/library/cache/redis"
"go-common/library/conf"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
"go-common/library/net/trace"
"github.com/BurntSushi/toml"
)
var (
confPath string
client *conf.Client
// Conf config
Conf = &Config{}
)
// Config .
type Config struct {
Log *log.Config
BM *bm.ServerConfig
Verify *verify.Config
Tracer *trace.Config
Redis *redis.Config
Ecode *ecode.Config
ItemCFJob *JobConfig
Hadoop *HadoopConfig
UserAreaJob *JobConfig
}
// HadoopConfig ...
type HadoopConfig struct {
HadoopDir string
TarUrl string
}
// JobConfig ...
type JobConfig struct {
Schedule string
// 多少个goroutine同时去写redis数据
WorkerNum int
// 如果指定了文件,使用这个文件
// 如果没有指定,自动去下载文件
// 可以是本地文件地址或者http文件地址
InputFile string
// 在hadoop里面文件的路径带日期
HadoopFile string
// Hadoop下载到本地的路径带日期
LocalTmpFile string
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init conf
func Init() error {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

View File

@@ -0,0 +1,32 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["dao.go"],
importpath = "go-common/app/job/live/recommend-job/internal/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/recommend-job/internal/conf:go_default_library",
"//library/cache/redis:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,37 @@
package dao
import (
"context"
"go-common/app/job/live/recommend-job/internal/conf"
"go-common/library/cache/redis"
)
// Dao dao
type Dao struct {
c *conf.Config
redis *redis.Pool
}
// New init mysql db
func New(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
redis: redis.NewPool(c.Redis),
}
return
}
// Close close the resource.
func (d *Dao) Close() {
d.redis.Close()
}
// Ping dao ping
func (d *Dao) Ping(ctx context.Context) error {
// TODO: add mc,redis... if you use
c := d.redis.Get(ctx)
defer c.Close()
_, err := c.Do("ping")
return err
}

View File

@@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["model.go"],
importpath = "go-common/app/job/live/recommend-job/internal/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1 @@
package model

View File

@@ -0,0 +1,35 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["http.go"],
importpath = "go-common/app/job/live/recommend-job/internal/server/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/recommend-job/internal/conf:go_default_library",
"//app/job/live/recommend-job/internal/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,53 @@
package http
import (
"net/http"
"go-common/app/job/live/recommend-job/internal/conf"
"go-common/app/job/live/recommend-job/internal/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
)
var (
vfy *verify.Verify
svc *service.Service
)
// Init init
func Init(c *conf.Config, s *service.Service) {
svc = s
vfy = verify.New(c.Verify)
engine := bm.DefaultServer(c.BM)
route(engine)
if err := engine.Start(); err != nil {
log.Error("bm Start error(%v)", err)
panic(err)
}
}
func route(e *bm.Engine) {
e.Ping(ping)
e.Register(register)
g := e.Group("/x/recommend")
{
g.GET("/start", vfy.Verify, howToStart)
}
}
func ping(ctx *bm.Context) {
if err := svc.Ping(ctx); err != nil {
log.Error("ping error(%v)", err)
ctx.AbortWithStatus(http.StatusServiceUnavailable)
}
}
func register(c *bm.Context) {
c.JSON(map[string]interface{}{}, nil)
}
// example for http request handler
func howToStart(c *bm.Context) {
c.String(0, "Golang 大法好 !!!")
}

View File

@@ -0,0 +1,43 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"item_cf_job.go",
"service.go",
"user_area_job.go",
],
importpath = "go-common/app/job/live/recommend-job/internal/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/recommend-job/internal/conf:go_default_library",
"//app/job/live/recommend-job/internal/dao:go_default_library",
"//app/service/live/recommend/recconst:go_default_library",
"//library/cache/redis:go_default_library",
"//library/log:go_default_library",
"//library/sync/errgroup:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/github.com/robfig/cron:go_default_library",
"//vendor/github.com/siddontang/go/ioutil2:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,340 @@
package service
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/siddontang/go/ioutil2"
"go-common/app/service/live/recommend/recconst"
"go-common/app/job/live/recommend-job/internal/conf"
"go-common/library/cache/redis"
"go-common/library/log"
"go-common/library/sync/errgroup"
"github.com/pkg/errors"
)
// ItemCFJob 把 hadoop 的数据写到redis
type ItemCFJob struct {
Conf *conf.JobConfig
RedisConf *redis.Config
HadoopConf *conf.HadoopConfig
}
// Run ...
func (j *ItemCFJob) Run() {
log.Info("Run ItemCFJob")
processFile(j.Conf, j.HadoopConf, j.RedisConf, writeItemCFToRedis)
log.Info("ItemCFJob Done")
}
func fetchFiles(jobConf *conf.JobConfig,
hadoopConf *conf.HadoopConfig) (paths []string, err error) {
path := jobConf.InputFile
if path == "" {
var javaHome string
var hadoopHome string
hadoopHome, javaHome, err = downloadHadoop(hadoopConf)
if err != nil {
log.Info("download hadoop err %+v ", err)
return
}
log.Info("download hadoop success: " + hadoopHome)
path, err = downloadFileFromHadoop(jobConf, hadoopHome, javaHome)
if err != nil {
log.Info("path %s downloaded err %+v ", path, err)
return
}
log.Info("path downloaded success: " + path)
var file os.FileInfo
file, err = os.Stat(path)
if err != nil {
log.Error("cannot open file %s err: %+v", file, err)
return
}
if file.IsDir() {
var files []os.FileInfo
files, err = ioutil.ReadDir(path)
if err != nil {
log.Error("no file in dir: %d, err: %+v", path, err)
return
}
for _, f := range files {
if strings.Index(f.Name(), ".") != 0 && strings.Index(f.Name(), "_") != 0 {
paths = append(paths, path+"/"+f.Name())
}
}
} else {
paths = []string{path}
}
} else {
if strings.Index(path, "http://") == 0 {
var httpPath = path
path, err = downloadFileFromHttp(httpPath, "/tmp/job-downloaded.txt")
if err != nil {
log.Error("download from http path=%s, error=%+v", httpPath, err)
return
}
log.Info("file downloaded from http %s to %s", httpPath, path)
} else {
var file os.FileInfo
file, err = os.Stat(path)
if err != nil {
log.Error("cannot open file %s err: %+v", file, err)
return
}
if file.IsDir() {
var files []os.FileInfo
files, err = ioutil.ReadDir(path)
if err != nil {
log.Error("no file in dir: %d, err: %+v", path, err)
return
}
for _, f := range files {
if strings.Index(f.Name(), ".") != 0 && strings.Index(f.Name(), "_") != 0 {
paths = append(paths, path+"/"+f.Name())
}
}
} else {
paths = []string{path}
}
}
}
return
}
// 按行读取文件并调用handler处理
func processFile(jobConf *conf.JobConfig,
hadoopConf *conf.HadoopConfig,
redisConf *redis.Config,
handler func(line string, pool *redis.Pool) error,
) {
paths, err := fetchFiles(jobConf, hadoopConf)
if err != nil {
return
}
var workerNum = jobConf.WorkerNum
var r = redis.NewPool(redisConf)
defer r.Close()
log.Info("all of files %+v", paths)
for _, path := range paths {
var startLineNum int
file, err := os.Open(path)
if err != nil {
log.Error("open path %s err %+v", path, errors.WithStack(err))
panic(err)
}
name := filepath.Base(filepath.Dir(path)) + "-" + filepath.Base(path)
offsetPath := "/tmp/" + name + ".offset"
// 读取上一次的位置
contentB, e := ioutil.ReadFile(offsetPath)
if e == nil {
content := string(contentB)
offset, e := strconv.Atoi(content)
if e == nil {
startLineNum = offset
} else {
startLineNum = 1
}
} else {
startLineNum = 1
}
log.Info("start from line: %d, file : %s offset file: %s", startLineNum, path, offsetPath)
var ops uint64
totalCount := lineCounter(path)
scanner := bufio.NewScanner(file)
g := errgroup.Group{}
g.GOMAXPROCS(workerNum)
var lineNum = 0
for scanner.Scan() {
lineNum++
if lineNum < startLineNum {
continue
}
line := scanner.Text() // 10001 [1:0.9,2:0.9]
g.Go(func() error {
handler(line, r)
atomic.AddUint64(&ops, 1)
if ops%20000 == 0 {
fmt.Printf("progress %d / %d percent %f %s \r", ops+uint64(startLineNum)-1, totalCount,
float32(uint64(startLineNum)+ops-1)/float32(totalCount)*100, "%")
} else if ops == uint64(totalCount) {
fmt.Printf("progress %d / %d 100%%\n", ops, totalCount)
}
return nil
})
if lineNum%100000 == 0 {
g.Wait()
ioutil.WriteFile(offsetPath,
[]byte(strconv.FormatInt(int64(lineNum), 10)),
os.ModePerm)
g = errgroup.Group{}
g.GOMAXPROCS(workerNum)
}
}
g.Wait()
if err = scanner.Err(); err != nil {
log.Error("err %+v", errors.WithStack(err))
panic(err)
}
file.Close()
}
}
func writeItemCFToRedis(line string, r *redis.Pool) (err error) {
start := strings.Index(line, "[")
end := strings.LastIndex(line, "]")
userIdStr := line[0 : start-1]
items := strings.Split(line[start+1:end], ",")
c := r.Get(context.Background())
defer c.Close()
userId, _ := strconv.Atoi(userIdStr)
key := fmt.Sprintf(recconst.UserItemCFRecKey, userId)
// 最多保留50个推荐
var inMemoryCount int
inMemoryCount, err = redis.Int(c.Do("ZCARD", key))
if err != nil {
log.Error("zcard err: %+v", err)
} else {
var toBeRemovedCount = inMemoryCount + len(items) - 60
if toBeRemovedCount > 0 {
var removed int
removed, err = redis.Int(c.Do("ZREMRANGEBYRANK", key, 0, toBeRemovedCount-1))
if err != nil {
log.Error("ZREMRANGEBYRANK key:%s, err: +%v", key, err)
} else {
log.Info("zset removed %d count, key:%s", removed, key)
}
}
}
for _, item := range items {
split := strings.Split(item, ":")
itemID := split[0]
score := split[1]
c.Send("ZADD", key, score, itemID)
}
c.Send("EXPIRE", key, 86400*30)
err = c.Flush()
if err != nil {
log.Error("zadd to redis error: %+v , key=%s", err, key)
return err
}
for i := 0; i < len(items)+1; i++ {
_, err = c.Receive()
if err != nil {
log.Error("zadd to redis error: %+v , key=%s, line=%s", err, key, line)
return err
}
}
return nil
}
func lineCounter(path string) int {
buf := make([]byte, 32*1024)
r, _ := os.Open(path)
defer r.Close()
count := 0
lineSep := []byte{'\n'}
for {
c, err := r.Read(buf)
count += bytes.Count(buf[:c], lineSep)
switch {
case err == io.EOF:
return count
case err != nil:
return count
}
}
}
func downloadHadoop(hadoopConf *conf.HadoopConfig) (hadoopHome string, javaHome string, err error) {
if strings.LastIndex(hadoopConf.HadoopDir, "/") == len(hadoopConf.HadoopDir)-1 {
hadoopHome = hadoopConf.HadoopDir + "hadoop-2.8.4"
} else {
hadoopHome = hadoopConf.HadoopDir + "/hadoop-2.8.4"
}
javaHome = hadoopHome + "/jdk1.8.0_60"
if ioutil2.FileExists(hadoopHome) {
return
}
var cmd = "curl -sSLf " + hadoopConf.TarUrl + " -o /tmp/hadoop.tar.gz"
err = runCmd(cmd)
if err != nil {
return
}
cmd = "tar -C " + hadoopConf.HadoopDir + " -xf /tmp/hadoop.tar.gz"
err = runCmd(cmd)
if err != nil {
return
}
return
}
func downloadFileFromHttp(url string, output string) (string, error) {
var localPath = output
var cmd = "curl -sSLf " + url + " -o " + localPath
var err = runCmd(cmd)
if err != nil {
return "", err
}
return localPath, nil
}
func downloadFileFromHadoop(jobConf *conf.JobConfig, hadoopHome string, javaHome string) (string, error) {
t := time.Now().AddDate(0, 0, -1)
day := t.Format("20060102")
localPath := fmt.Sprintf(jobConf.LocalTmpFile, day)
if ioutil2.FileExists(localPath) {
return localPath, nil
}
remotePath := fmt.Sprintf(jobConf.HadoopFile, day)
cmd := fmt.Sprintf("export JAVA_HOME=%s; %s/bin/hdfs dfs -get %s %s",
javaHome, hadoopHome, remotePath, localPath)
err := runCmd(cmd)
return localPath, err
}
// runCmd runs the cmd & print output (both stdout & stderr)
func runCmd(cmd string) (err error) {
fmt.Printf("CMD: %s \n", cmd)
out, err := exec.Command("/bin/bash", "-c", cmd).CombinedOutput()
log.Info(string(out))
if err != nil {
err = errors.Wrap(err, string(out))
}
return
}

View File

@@ -0,0 +1,50 @@
package service
import (
"context"
"go-common/app/job/live/recommend-job/internal/conf"
"go-common/app/job/live/recommend-job/internal/dao"
"github.com/robfig/cron"
)
// Service struct
type Service struct {
c *conf.Config
dao *dao.Dao
cron *cron.Cron
}
// New init
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: dao.New(c),
}
return s
}
// RunCrontab ...
func (s *Service) RunCrontab() {
s.cron = cron.New()
if s.c.ItemCFJob.Schedule == "" {
panic("invalid schedule: " + s.c.ItemCFJob.Schedule)
}
if s.c.UserAreaJob.Schedule == "" {
panic("invalid schedule: " + s.c.UserAreaJob.Schedule)
}
s.cron.AddJob(s.c.ItemCFJob.Schedule, &ItemCFJob{Conf: s.c.ItemCFJob, RedisConf: s.c.Redis, HadoopConf: s.c.Hadoop})
s.cron.AddJob(s.c.UserAreaJob.Schedule, &UserAreaJob{JobConf: s.c.UserAreaJob, RedisConf: s.c.Redis, HadoopConf: s.c.Hadoop})
s.cron.Start()
}
// Ping Service
func (s *Service) Ping(ctx context.Context) (err error) {
return s.dao.Ping(ctx)
}
// Close Service
func (s *Service) Close() {
s.dao.Close()
}

View File

@@ -0,0 +1,45 @@
package service
import (
"context"
"fmt"
"strconv"
"strings"
"go-common/app/job/live/recommend-job/internal/conf"
"go-common/app/service/live/recommend/recconst"
"go-common/library/cache/redis"
"go-common/library/log"
)
// UserAreaJob 把用户分区偏好导入到redis
type UserAreaJob struct {
JobConf *conf.JobConfig
RedisConf *redis.Config
HadoopConf *conf.HadoopConfig
}
// Run ...
func (j *UserAreaJob) Run() {
log.Info("UserAreaJob Start")
processFile(j.JobConf, j.HadoopConf, j.RedisConf, writeUserAreaToRedis)
log.Info("UserAreaJob End")
}
func writeUserAreaToRedis(line string, pool *redis.Pool) (err error) {
var split = strings.Split(line, ",")
var uid = split[0]
uid = strings.Trim(uid, "\"")
var areaIds = split[1]
areaIds = strings.Trim(areaIds, "\"")
var ctx = context.Background()
var conn = pool.Get(ctx)
defer conn.Close()
uidInt, _ := strconv.Atoi(uid)
var key = fmt.Sprintf(recconst.UserAreaKey, uidInt)
_, err = conn.Do("SETEX", key, 86400*7, areaIds)
if err != nil {
log.Error("writeUserAreaToRedis err +%v, key=%s", err, key)
}
return
}