Create & Init Project...

app/job/live/recommend-job/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/live/recommend-job/cmd:all-srcs",
+        "//app/job/live/recommend-job/internal/conf:all-srcs",
+        "//app/job/live/recommend-job/internal/dao:all-srcs",
+        "//app/job/live/recommend-job/internal/model:all-srcs",
+        "//app/job/live/recommend-job/internal/server/http:all-srcs",
+        "//app/job/live/recommend-job/internal/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/recommend-job/CHANGELOG.md

@@ -0,0 +1,2 @@
+### v1.0.0
+1. 上线功能xxx

app/job/live/recommend-job/CONTRIBUTORS.md

@@ -0,0 +1,6 @@
+# Owner
+liugang
+
+# Author
+
+# Reviewer

app/job/live/recommend-job/OWNERS

@@ -0,0 +1,10 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- liugang
+labels:
+- job
+- job/live/recommend-job
+- live
+options:
+  no_parent_owners: true

app/job/live/recommend-job/README.md

@@ -0,0 +1,22 @@
+# live-web-interface
+
+# 项目简介
+推荐系统的离线脚本服务
+
+# 编译环境
+
+
+# 依赖包
+
+
+# 编译执行
+
+# 部署说明
+必须从hadoop拉取数据文件，所以需要hadoop客户端
+
+hadoop客户端从 http://livedev.bilibili.co/download/hadoop-sz.tar.gz 拉取
+
+文件服务部署在 shylf-live-app-27
+
+对于uat环境，
+没有分布式hadoop，部署一个http的服务提供单机hadoop的文件下载

app/job/live/recommend-job/api/http/api.md

@@ -0,0 +1 @@
+# HTTP API文档

app/job/live/recommend-job/cmd/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["test.toml"],
+    importpath = "go-common/app/job/live/recommend-job/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/recommend-job/internal/conf:go_default_library",
+        "//app/job/live/recommend-job/internal/server/http:go_default_library",
+        "//app/job/live/recommend-job/internal/service:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/recommend-job/cmd/main.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/live/recommend-job/internal/conf"
+	"go-common/app/job/live/recommend-job/internal/server/http"
+	"go-common/app/job/live/recommend-job/internal/service"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+var itemCFRun = false
+var itemCFInputPath = ""
+var itemCFWorkerNum = 1000
+
+var userAreaRun = false
+var uerAreaInputPath = ""
+
+func main() {
+	flag.BoolVar(&userAreaRun, "userarea.run", false, "用户分区缓存：是否手动跑")
+	flag.StringVar(&uerAreaInputPath, "userarea.input", "", "用户分区缓存：输入文件地址")
+
+	flag.BoolVar(&itemCFRun, "itemcf.run", false, "协同过滤推荐缓存到redis：是否手动跑")
+	flag.StringVar(&itemCFInputPath, "itemcf.input", "", "协同过滤结果文件地址")
+	flag.IntVar(&itemCFWorkerNum, "itemcf.workerNum", 1000, "worker数量")
+
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("recommend-job start")
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	ecode.Init(conf.Conf.Ecode)
+
+	if itemCFRun {
+		c := conf.Conf.ItemCFJob
+		c.InputFile = itemCFInputPath
+		c.WorkerNum = itemCFWorkerNum
+		var job = service.ItemCFJob{
+			Conf:       c,
+			RedisConf:  conf.Conf.Redis,
+			HadoopConf: conf.Conf.Hadoop,
+		}
+		job.Run()
+		os.Exit(0)
+	}
+
+	if userAreaRun {
+		c := conf.Conf.UserAreaJob
+		c.InputFile = uerAreaInputPath
+		c.WorkerNum = 1000
+		var job = service.UserAreaJob{
+			JobConf:    c,
+			RedisConf:  conf.Conf.Redis,
+			HadoopConf: conf.Conf.Hadoop,
+		}
+		job.Run()
+		os.Exit(0)
+	}
+
+	svc := service.New(conf.Conf)
+	http.Init(conf.Conf, svc)
+	svc.RunCrontab()
+
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			svc.Close()
+			log.Info("recommend-job exit")
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/live/recommend-job/cmd/test.toml

@@ -0,0 +1,31 @@
+[redis]
+	name = "recommend-job"
+	proto = "tcp"
+	addr = "127.0.0.1:6379"
+	idle = 2000
+	active = 2000
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "1m"
+
+[log]
+    stdout = true
+[hadoop]
+    tarUrl = "http://livedev.bilibili.co/download/hadoop-2.8.4.tar.gz"
+    hadoopDir = "/tmp"
+
+[itemcfJob]
+    schedule = "30 50 16 * * *" # 秒 分钟 小时 天 月 星期
+    workerNum = 1000
+    inputFile = "/department/live/recommend/item-cf-output"
+    hadoopFile = "/department/live/recommend/%s/item-cf-output"
+    localTmpFile = "/tmp/recommend-cf-output-%s"
+
+[userAreaJob]
+    schedule = "20 37 14 * * *" # 秒 分钟 小时 天 月 星期
+    workerNum = 1000
+    inputFile = "/Users/liugang/Downloads/user-area.txt"
+    hadoopFile = "/department/live/recommend/%s/user-area"
+    localTmpFile = "/tmp/user-area-output-%s"

app/job/live/recommend-job/internal/conf/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/live/recommend-job/internal/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/recommend-job/internal/conf/conf.go

@@ -0,0 +1,108 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+	"go-common/library/net/trace"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config .
+type Config struct {
+	Log         *log.Config
+	BM          *bm.ServerConfig
+	Verify      *verify.Config
+	Tracer      *trace.Config
+	Redis       *redis.Config
+	Ecode       *ecode.Config
+	ItemCFJob   *JobConfig
+	Hadoop      *HadoopConfig
+	UserAreaJob *JobConfig
+}
+
+// HadoopConfig ...
+type HadoopConfig struct {
+	HadoopDir string
+	TarUrl    string
+}
+
+// JobConfig ...
+type JobConfig struct {
+	Schedule string
+	// 多少个goroutine同时去写redis数据
+	WorkerNum int
+	// 如果指定了文件，使用这个文件
+	// 如果没有指定，自动去下载文件
+	// 可以是本地文件地址，或者http文件地址
+	InputFile string
+	// 在hadoop里面文件的路径，带日期
+	HadoopFile string
+	// Hadoop下载到本地的路径，带日期
+	LocalTmpFile string
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/live/recommend-job/internal/dao/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dao.go"],
+    importpath = "go-common/app/job/live/recommend-job/internal/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/recommend-job/internal/conf:go_default_library",
+        "//library/cache/redis:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/recommend-job/internal/dao/dao.go

@@ -0,0 +1,37 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/live/recommend-job/internal/conf"
+	"go-common/library/cache/redis"
+)
+
+// Dao dao
+type Dao struct {
+	c     *conf.Config
+	redis *redis.Pool
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:     c,
+		redis: redis.NewPool(c.Redis),
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(ctx context.Context) error {
+	// TODO: add mc,redis... if you use
+	c := d.redis.Get(ctx)
+	defer c.Close()
+	_, err := c.Do("ping")
+	return err
+}

app/job/live/recommend-job/internal/model/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/job/live/recommend-job/internal/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/recommend-job/internal/model/model.go

@@ -0,0 +1 @@
+package model

app/job/live/recommend-job/internal/server/http/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/live/recommend-job/internal/server/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/recommend-job/internal/conf:go_default_library",
+        "//app/job/live/recommend-job/internal/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/recommend-job/internal/server/http/http.go

@@ -0,0 +1,53 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/live/recommend-job/internal/conf"
+	"go-common/app/job/live/recommend-job/internal/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+)
+
+var (
+	vfy *verify.Verify
+	svc *service.Service
+)
+
+// Init init
+func Init(c *conf.Config, s *service.Service) {
+	svc = s
+	vfy = verify.New(c.Verify)
+	engine := bm.DefaultServer(c.BM)
+	route(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("bm Start error(%v)", err)
+		panic(err)
+	}
+}
+
+func route(e *bm.Engine) {
+	e.Ping(ping)
+	e.Register(register)
+	g := e.Group("/x/recommend")
+	{
+		g.GET("/start", vfy.Verify, howToStart)
+	}
+}
+
+func ping(ctx *bm.Context) {
+	if err := svc.Ping(ctx); err != nil {
+		log.Error("ping error(%v)", err)
+		ctx.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+func register(c *bm.Context) {
+	c.JSON(map[string]interface{}{}, nil)
+}
+
+// example for http request handler
+func howToStart(c *bm.Context) {
+	c.String(0, "Golang 大法好 !!!")
+}

app/job/live/recommend-job/internal/service/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "item_cf_job.go",
+        "service.go",
+        "user_area_job.go",
+    ],
+    importpath = "go-common/app/job/live/recommend-job/internal/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/recommend-job/internal/conf:go_default_library",
+        "//app/job/live/recommend-job/internal/dao:go_default_library",
+        "//app/service/live/recommend/recconst:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+        "//vendor/github.com/siddontang/go/ioutil2:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/recommend-job/internal/service/item_cf_job.go

@@ -0,0 +1,340 @@
+package service
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/siddontang/go/ioutil2"
+
+	"go-common/app/service/live/recommend/recconst"
+
+	"go-common/app/job/live/recommend-job/internal/conf"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+
+	"github.com/pkg/errors"
+)
+
+// ItemCFJob 把 hadoop 的数据写到redis
+type ItemCFJob struct {
+	Conf       *conf.JobConfig
+	RedisConf  *redis.Config
+	HadoopConf *conf.HadoopConfig
+}
+
+// Run ...
+func (j *ItemCFJob) Run() {
+	log.Info("Run ItemCFJob")
+	processFile(j.Conf, j.HadoopConf, j.RedisConf, writeItemCFToRedis)
+	log.Info("ItemCFJob Done")
+}
+
+func fetchFiles(jobConf *conf.JobConfig,
+	hadoopConf *conf.HadoopConfig) (paths []string, err error) {
+	path := jobConf.InputFile
+	if path == "" {
+		var javaHome string
+		var hadoopHome string
+		hadoopHome, javaHome, err = downloadHadoop(hadoopConf)
+		if err != nil {
+			log.Info("download hadoop err %+v ", err)
+			return
+		}
+		log.Info("download hadoop success: " + hadoopHome)
+		path, err = downloadFileFromHadoop(jobConf, hadoopHome, javaHome)
+		if err != nil {
+			log.Info("path %s downloaded err %+v  ", path, err)
+			return
+		}
+		log.Info("path downloaded success: " + path)
+
+		var file os.FileInfo
+		file, err = os.Stat(path)
+		if err != nil {
+			log.Error("cannot open file %s err: %+v", file, err)
+			return
+		}
+		if file.IsDir() {
+			var files []os.FileInfo
+			files, err = ioutil.ReadDir(path)
+			if err != nil {
+				log.Error("no file in dir: %d, err: %+v", path, err)
+				return
+			}
+
+			for _, f := range files {
+				if strings.Index(f.Name(), ".") != 0 && strings.Index(f.Name(), "_") != 0 {
+					paths = append(paths, path+"/"+f.Name())
+				}
+			}
+		} else {
+			paths = []string{path}
+		}
+	} else {
+		if strings.Index(path, "http://") == 0 {
+			var httpPath = path
+			path, err = downloadFileFromHttp(httpPath, "/tmp/job-downloaded.txt")
+			if err != nil {
+				log.Error("download from http path=%s, error=%+v", httpPath, err)
+				return
+			}
+			log.Info("file downloaded from http %s to %s", httpPath, path)
+		} else {
+			var file os.FileInfo
+			file, err = os.Stat(path)
+			if err != nil {
+				log.Error("cannot open file %s err: %+v", file, err)
+				return
+			}
+			if file.IsDir() {
+				var files []os.FileInfo
+				files, err = ioutil.ReadDir(path)
+				if err != nil {
+					log.Error("no file in dir: %d, err: %+v", path, err)
+					return
+				}
+
+				for _, f := range files {
+					if strings.Index(f.Name(), ".") != 0 && strings.Index(f.Name(), "_") != 0 {
+						paths = append(paths, path+"/"+f.Name())
+					}
+				}
+			} else {
+				paths = []string{path}
+			}
+		}
+	}
+	return
+}
+
+// 按行读取文件，并调用handler处理
+func processFile(jobConf *conf.JobConfig,
+	hadoopConf *conf.HadoopConfig,
+	redisConf *redis.Config,
+	handler func(line string, pool *redis.Pool) error,
+) {
+	paths, err := fetchFiles(jobConf, hadoopConf)
+	if err != nil {
+		return
+	}
+	var workerNum = jobConf.WorkerNum
+	var r = redis.NewPool(redisConf)
+	defer r.Close()
+
+	log.Info("all of files %+v", paths)
+	for _, path := range paths {
+
+		var startLineNum int
+
+		file, err := os.Open(path)
+		if err != nil {
+			log.Error("open path %s err %+v", path, errors.WithStack(err))
+			panic(err)
+		}
+
+		name := filepath.Base(filepath.Dir(path)) + "-" + filepath.Base(path)
+		offsetPath := "/tmp/" + name + ".offset"
+		// 读取上一次的位置
+		contentB, e := ioutil.ReadFile(offsetPath)
+		if e == nil {
+			content := string(contentB)
+			offset, e := strconv.Atoi(content)
+			if e == nil {
+				startLineNum = offset
+			} else {
+				startLineNum = 1
+			}
+		} else {
+			startLineNum = 1
+		}
+		log.Info("start from line: %d, file : %s offset file: %s", startLineNum, path, offsetPath)
+
+		var ops uint64
+		totalCount := lineCounter(path)
+		scanner := bufio.NewScanner(file)
+
+		g := errgroup.Group{}
+		g.GOMAXPROCS(workerNum)
+
+		var lineNum = 0
+		for scanner.Scan() {
+			lineNum++
+			if lineNum < startLineNum {
+				continue
+			}
+
+			line := scanner.Text() // 10001 [1:0.9,2:0.9]
+			g.Go(func() error {
+				handler(line, r)
+				atomic.AddUint64(&ops, 1)
+				if ops%20000 == 0 {
+					fmt.Printf("progress %d / %d percent %f %s         \r", ops+uint64(startLineNum)-1, totalCount,
+						float32(uint64(startLineNum)+ops-1)/float32(totalCount)*100, "%")
+				} else if ops == uint64(totalCount) {
+					fmt.Printf("progress %d / %d 100%%\n", ops, totalCount)
+				}
+				return nil
+			})
+			if lineNum%100000 == 0 {
+				g.Wait()
+				ioutil.WriteFile(offsetPath,
+					[]byte(strconv.FormatInt(int64(lineNum), 10)),
+					os.ModePerm)
+				g = errgroup.Group{}
+				g.GOMAXPROCS(workerNum)
+			}
+		}
+
+		g.Wait()
+
+		if err = scanner.Err(); err != nil {
+			log.Error("err %+v", errors.WithStack(err))
+			panic(err)
+		}
+		file.Close()
+	}
+
+}
+
+func writeItemCFToRedis(line string, r *redis.Pool) (err error) {
+	start := strings.Index(line, "[")
+	end := strings.LastIndex(line, "]")
+	userIdStr := line[0 : start-1]
+	items := strings.Split(line[start+1:end], ",")
+	c := r.Get(context.Background())
+	defer c.Close()
+	userId, _ := strconv.Atoi(userIdStr)
+	key := fmt.Sprintf(recconst.UserItemCFRecKey, userId)
+	// 最多保留50个推荐
+	var inMemoryCount int
+	inMemoryCount, err = redis.Int(c.Do("ZCARD", key))
+	if err != nil {
+		log.Error("zcard err: %+v", err)
+	} else {
+		var toBeRemovedCount = inMemoryCount + len(items) - 60
+		if toBeRemovedCount > 0 {
+			var removed int
+			removed, err = redis.Int(c.Do("ZREMRANGEBYRANK", key, 0, toBeRemovedCount-1))
+			if err != nil {
+				log.Error("ZREMRANGEBYRANK key:%s, err: +%v", key, err)
+			} else {
+				log.Info("zset removed %d count, key:%s", removed, key)
+			}
+		}
+	}
+	for _, item := range items {
+		split := strings.Split(item, ":")
+		itemID := split[0]
+		score := split[1]
+
+		c.Send("ZADD", key, score, itemID)
+	}
+	c.Send("EXPIRE", key, 86400*30)
+	err = c.Flush()
+
+	if err != nil {
+		log.Error("zadd to redis error: %+v , key=%s", err, key)
+		return err
+	}
+	for i := 0; i < len(items)+1; i++ {
+		_, err = c.Receive()
+		if err != nil {
+			log.Error("zadd to redis error: %+v , key=%s, line=%s", err, key, line)
+			return err
+		}
+	}
+	return nil
+}
+
+func lineCounter(path string) int {
+	buf := make([]byte, 32*1024)
+	r, _ := os.Open(path)
+	defer r.Close()
+	count := 0
+	lineSep := []byte{'\n'}
+
+	for {
+		c, err := r.Read(buf)
+		count += bytes.Count(buf[:c], lineSep)
+
+		switch {
+		case err == io.EOF:
+			return count
+
+		case err != nil:
+			return count
+		}
+	}
+}
+
+func downloadHadoop(hadoopConf *conf.HadoopConfig) (hadoopHome string, javaHome string, err error) {
+	if strings.LastIndex(hadoopConf.HadoopDir, "/") == len(hadoopConf.HadoopDir)-1 {
+		hadoopHome = hadoopConf.HadoopDir + "hadoop-2.8.4"
+	} else {
+		hadoopHome = hadoopConf.HadoopDir + "/hadoop-2.8.4"
+	}
+	javaHome = hadoopHome + "/jdk1.8.0_60"
+
+	if ioutil2.FileExists(hadoopHome) {
+		return
+	}
+	var cmd = "curl -sSLf " + hadoopConf.TarUrl + " -o /tmp/hadoop.tar.gz"
+	err = runCmd(cmd)
+	if err != nil {
+		return
+	}
+	cmd = "tar -C " + hadoopConf.HadoopDir + " -xf /tmp/hadoop.tar.gz"
+	err = runCmd(cmd)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func downloadFileFromHttp(url string, output string) (string, error) {
+	var localPath = output
+	var cmd = "curl -sSLf " + url + " -o " + localPath
+	var err = runCmd(cmd)
+	if err != nil {
+		return "", err
+	}
+	return localPath, nil
+}
+
+func downloadFileFromHadoop(jobConf *conf.JobConfig, hadoopHome string, javaHome string) (string, error) {
+	t := time.Now().AddDate(0, 0, -1)
+	day := t.Format("20060102")
+	localPath := fmt.Sprintf(jobConf.LocalTmpFile, day)
+	if ioutil2.FileExists(localPath) {
+		return localPath, nil
+	}
+	remotePath := fmt.Sprintf(jobConf.HadoopFile, day)
+	cmd := fmt.Sprintf("export JAVA_HOME=%s; %s/bin/hdfs dfs -get %s %s",
+		javaHome, hadoopHome, remotePath, localPath)
+	err := runCmd(cmd)
+	return localPath, err
+}
+
+// runCmd runs the cmd & print output (both stdout & stderr)
+func runCmd(cmd string) (err error) {
+	fmt.Printf("CMD: %s \n", cmd)
+	out, err := exec.Command("/bin/bash", "-c", cmd).CombinedOutput()
+	log.Info(string(out))
+	if err != nil {
+		err = errors.Wrap(err, string(out))
+	}
+	return
+}

app/job/live/recommend-job/internal/service/service.go

@@ -0,0 +1,50 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/live/recommend-job/internal/conf"
+	"go-common/app/job/live/recommend-job/internal/dao"
+
+	"github.com/robfig/cron"
+)
+
+// Service struct
+type Service struct {
+	c    *conf.Config
+	dao  *dao.Dao
+	cron *cron.Cron
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:   c,
+		dao: dao.New(c),
+	}
+	return s
+}
+
+// RunCrontab ...
+func (s *Service) RunCrontab() {
+	s.cron = cron.New()
+	if s.c.ItemCFJob.Schedule == "" {
+		panic("invalid schedule: " + s.c.ItemCFJob.Schedule)
+	}
+	if s.c.UserAreaJob.Schedule == "" {
+		panic("invalid schedule: " + s.c.UserAreaJob.Schedule)
+	}
+	s.cron.AddJob(s.c.ItemCFJob.Schedule, &ItemCFJob{Conf: s.c.ItemCFJob, RedisConf: s.c.Redis, HadoopConf: s.c.Hadoop})
+	s.cron.AddJob(s.c.UserAreaJob.Schedule, &UserAreaJob{JobConf: s.c.UserAreaJob, RedisConf: s.c.Redis, HadoopConf: s.c.Hadoop})
+	s.cron.Start()
+}
+
+// Ping Service
+func (s *Service) Ping(ctx context.Context) (err error) {
+	return s.dao.Ping(ctx)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/job/live/recommend-job/internal/service/user_area_job.go

@@ -0,0 +1,45 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"go-common/app/job/live/recommend-job/internal/conf"
+	"go-common/app/service/live/recommend/recconst"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+// UserAreaJob 把用户分区偏好导入到redis
+type UserAreaJob struct {
+	JobConf    *conf.JobConfig
+	RedisConf  *redis.Config
+	HadoopConf *conf.HadoopConfig
+}
+
+// Run ...
+func (j *UserAreaJob) Run() {
+	log.Info("UserAreaJob Start")
+	processFile(j.JobConf, j.HadoopConf, j.RedisConf, writeUserAreaToRedis)
+	log.Info("UserAreaJob End")
+}
+
+func writeUserAreaToRedis(line string, pool *redis.Pool) (err error) {
+	var split = strings.Split(line, ",")
+	var uid = split[0]
+	uid = strings.Trim(uid, "\"")
+	var areaIds = split[1]
+	areaIds = strings.Trim(areaIds, "\"")
+	var ctx = context.Background()
+	var conn = pool.Get(ctx)
+	defer conn.Close()
+	uidInt, _ := strconv.Atoi(uid)
+	var key = fmt.Sprintf(recconst.UserAreaKey, uidInt)
+	_, err = conn.Do("SETEX", key, 86400*7, areaIds)
+	if err != nil {
+		log.Error("writeUserAreaToRedis err +%v, key=%s", err, key)
+	}
+	return
+}