341 lines
8.0 KiB
Go
341 lines
8.0 KiB
Go
|
package service
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"bufio"
|
|||
|
|
"bytes"
|
|||
|
|
"context"
|
|||
|
|
"fmt"
|
|||
|
|
"io"
|
|||
|
|
"io/ioutil"
|
|||
|
|
"os"
|
|||
|
|
"os/exec"
|
|||
|
|
"path/filepath"
|
|||
|
|
"strconv"
|
|||
|
|
"strings"
|
|||
|
|
"sync/atomic"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"github.com/siddontang/go/ioutil2"
|
|||
|
|
|
|||
|
|
"go-common/app/service/live/recommend/recconst"
|
|||
|
|
|
|||
|
|
"go-common/app/job/live/recommend-job/internal/conf"
|
|||
|
|
"go-common/library/cache/redis"
|
|||
|
|
"go-common/library/log"
|
|||
|
|
"go-common/library/sync/errgroup"
|
|||
|
|
|
|||
|
|
"github.com/pkg/errors"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// ItemCFJob 把 hadoop 的数据写到redis
|
|||
|
|
type ItemCFJob struct {
|
|||
|
|
Conf *conf.JobConfig
|
|||
|
|
RedisConf *redis.Config
|
|||
|
|
HadoopConf *conf.HadoopConfig
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Run ...
|
|||
|
|
func (j *ItemCFJob) Run() {
|
|||
|
|
log.Info("Run ItemCFJob")
|
|||
|
|
processFile(j.Conf, j.HadoopConf, j.RedisConf, writeItemCFToRedis)
|
|||
|
|
log.Info("ItemCFJob Done")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func fetchFiles(jobConf *conf.JobConfig,
|
|||
|
|
hadoopConf *conf.HadoopConfig) (paths []string, err error) {
|
|||
|
|
path := jobConf.InputFile
|
|||
|
|
if path == "" {
|
|||
|
|
var javaHome string
|
|||
|
|
var hadoopHome string
|
|||
|
|
hadoopHome, javaHome, err = downloadHadoop(hadoopConf)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Info("download hadoop err %+v ", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
log.Info("download hadoop success: " + hadoopHome)
|
|||
|
|
path, err = downloadFileFromHadoop(jobConf, hadoopHome, javaHome)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Info("path %s downloaded err %+v ", path, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
log.Info("path downloaded success: " + path)
|
|||
|
|
|
|||
|
|
var file os.FileInfo
|
|||
|
|
file, err = os.Stat(path)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("cannot open file %s err: %+v", file, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if file.IsDir() {
|
|||
|
|
var files []os.FileInfo
|
|||
|
|
files, err = ioutil.ReadDir(path)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("no file in dir: %d, err: %+v", path, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, f := range files {
|
|||
|
|
if strings.Index(f.Name(), ".") != 0 && strings.Index(f.Name(), "_") != 0 {
|
|||
|
|
paths = append(paths, path+"/"+f.Name())
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
paths = []string{path}
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
if strings.Index(path, "http://") == 0 {
|
|||
|
|
var httpPath = path
|
|||
|
|
path, err = downloadFileFromHttp(httpPath, "/tmp/job-downloaded.txt")
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("download from http path=%s, error=%+v", httpPath, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
log.Info("file downloaded from http %s to %s", httpPath, path)
|
|||
|
|
} else {
|
|||
|
|
var file os.FileInfo
|
|||
|
|
file, err = os.Stat(path)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("cannot open file %s err: %+v", file, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if file.IsDir() {
|
|||
|
|
var files []os.FileInfo
|
|||
|
|
files, err = ioutil.ReadDir(path)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("no file in dir: %d, err: %+v", path, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, f := range files {
|
|||
|
|
if strings.Index(f.Name(), ".") != 0 && strings.Index(f.Name(), "_") != 0 {
|
|||
|
|
paths = append(paths, path+"/"+f.Name())
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
paths = []string{path}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 按行读取文件，并调用handler处理
|
|||
|
|
func processFile(jobConf *conf.JobConfig,
|
|||
|
|
hadoopConf *conf.HadoopConfig,
|
|||
|
|
redisConf *redis.Config,
|
|||
|
|
handler func(line string, pool *redis.Pool) error,
|
|||
|
|
) {
|
|||
|
|
paths, err := fetchFiles(jobConf, hadoopConf)
|
|||
|
|
if err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
var workerNum = jobConf.WorkerNum
|
|||
|
|
var r = redis.NewPool(redisConf)
|
|||
|
|
defer r.Close()
|
|||
|
|
|
|||
|
|
log.Info("all of files %+v", paths)
|
|||
|
|
for _, path := range paths {
|
|||
|
|
|
|||
|
|
var startLineNum int
|
|||
|
|
|
|||
|
|
file, err := os.Open(path)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("open path %s err %+v", path, errors.WithStack(err))
|
|||
|
|
panic(err)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
name := filepath.Base(filepath.Dir(path)) + "-" + filepath.Base(path)
|
|||
|
|
offsetPath := "/tmp/" + name + ".offset"
|
|||
|
|
// 读取上一次的位置
|
|||
|
|
contentB, e := ioutil.ReadFile(offsetPath)
|
|||
|
|
if e == nil {
|
|||
|
|
content := string(contentB)
|
|||
|
|
offset, e := strconv.Atoi(content)
|
|||
|
|
if e == nil {
|
|||
|
|
startLineNum = offset
|
|||
|
|
} else {
|
|||
|
|
startLineNum = 1
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
startLineNum = 1
|
|||
|
|
}
|
|||
|
|
log.Info("start from line: %d, file : %s offset file: %s", startLineNum, path, offsetPath)
|
|||
|
|
|
|||
|
|
var ops uint64
|
|||
|
|
totalCount := lineCounter(path)
|
|||
|
|
scanner := bufio.NewScanner(file)
|
|||
|
|
|
|||
|
|
g := errgroup.Group{}
|
|||
|
|
g.GOMAXPROCS(workerNum)
|
|||
|
|
|
|||
|
|
var lineNum = 0
|
|||
|
|
for scanner.Scan() {
|
|||
|
|
lineNum++
|
|||
|
|
if lineNum < startLineNum {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
line := scanner.Text() // 10001 [1:0.9,2:0.9]
|
|||
|
|
g.Go(func() error {
|
|||
|
|
handler(line, r)
|
|||
|
|
atomic.AddUint64(&ops, 1)
|
|||
|
|
if ops%20000 == 0 {
|
|||
|
|
fmt.Printf("progress %d / %d percent %f %s \r", ops+uint64(startLineNum)-1, totalCount,
|
|||
|
|
float32(uint64(startLineNum)+ops-1)/float32(totalCount)*100, "%")
|
|||
|
|
} else if ops == uint64(totalCount) {
|
|||
|
|
fmt.Printf("progress %d / %d 100%%\n", ops, totalCount)
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
})
|
|||
|
|
if lineNum%100000 == 0 {
|
|||
|
|
g.Wait()
|
|||
|
|
ioutil.WriteFile(offsetPath,
|
|||
|
|
[]byte(strconv.FormatInt(int64(lineNum), 10)),
|
|||
|
|
os.ModePerm)
|
|||
|
|
g = errgroup.Group{}
|
|||
|
|
g.GOMAXPROCS(workerNum)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
g.Wait()
|
|||
|
|
|
|||
|
|
if err = scanner.Err(); err != nil {
|
|||
|
|
log.Error("err %+v", errors.WithStack(err))
|
|||
|
|
panic(err)
|
|||
|
|
}
|
|||
|
|
file.Close()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func writeItemCFToRedis(line string, r *redis.Pool) (err error) {
|
|||
|
|
start := strings.Index(line, "[")
|
|||
|
|
end := strings.LastIndex(line, "]")
|
|||
|
|
userIdStr := line[0 : start-1]
|
|||
|
|
items := strings.Split(line[start+1:end], ",")
|
|||
|
|
c := r.Get(context.Background())
|
|||
|
|
defer c.Close()
|
|||
|
|
userId, _ := strconv.Atoi(userIdStr)
|
|||
|
|
key := fmt.Sprintf(recconst.UserItemCFRecKey, userId)
|
|||
|
|
// 最多保留50个推荐
|
|||
|
|
var inMemoryCount int
|
|||
|
|
inMemoryCount, err = redis.Int(c.Do("ZCARD", key))
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("zcard err: %+v", err)
|
|||
|
|
} else {
|
|||
|
|
var toBeRemovedCount = inMemoryCount + len(items) - 60
|
|||
|
|
if toBeRemovedCount > 0 {
|
|||
|
|
var removed int
|
|||
|
|
removed, err = redis.Int(c.Do("ZREMRANGEBYRANK", key, 0, toBeRemovedCount-1))
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("ZREMRANGEBYRANK key:%s, err: +%v", key, err)
|
|||
|
|
} else {
|
|||
|
|
log.Info("zset removed %d count, key:%s", removed, key)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, item := range items {
|
|||
|
|
split := strings.Split(item, ":")
|
|||
|
|
itemID := split[0]
|
|||
|
|
score := split[1]
|
|||
|
|
|
|||
|
|
c.Send("ZADD", key, score, itemID)
|
|||
|
|
}
|
|||
|
|
c.Send("EXPIRE", key, 86400*30)
|
|||
|
|
err = c.Flush()
|
|||
|
|
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("zadd to redis error: %+v , key=%s", err, key)
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
for i := 0; i < len(items)+1; i++ {
|
|||
|
|
_, err = c.Receive()
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("zadd to redis error: %+v , key=%s, line=%s", err, key, line)
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func lineCounter(path string) int {
|
|||
|
|
buf := make([]byte, 32*1024)
|
|||
|
|
r, _ := os.Open(path)
|
|||
|
|
defer r.Close()
|
|||
|
|
count := 0
|
|||
|
|
lineSep := []byte{'\n'}
|
|||
|
|
|
|||
|
|
for {
|
|||
|
|
c, err := r.Read(buf)
|
|||
|
|
count += bytes.Count(buf[:c], lineSep)
|
|||
|
|
|
|||
|
|
switch {
|
|||
|
|
case err == io.EOF:
|
|||
|
|
return count
|
|||
|
|
|
|||
|
|
case err != nil:
|
|||
|
|
return count
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func downloadHadoop(hadoopConf *conf.HadoopConfig) (hadoopHome string, javaHome string, err error) {
|
|||
|
|
if strings.LastIndex(hadoopConf.HadoopDir, "/") == len(hadoopConf.HadoopDir)-1 {
|
|||
|
|
hadoopHome = hadoopConf.HadoopDir + "hadoop-2.8.4"
|
|||
|
|
} else {
|
|||
|
|
hadoopHome = hadoopConf.HadoopDir + "/hadoop-2.8.4"
|
|||
|
|
}
|
|||
|
|
javaHome = hadoopHome + "/jdk1.8.0_60"
|
|||
|
|
|
|||
|
|
if ioutil2.FileExists(hadoopHome) {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
var cmd = "curl -sSLf " + hadoopConf.TarUrl + " -o /tmp/hadoop.tar.gz"
|
|||
|
|
err = runCmd(cmd)
|
|||
|
|
if err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
cmd = "tar -C " + hadoopConf.HadoopDir + " -xf /tmp/hadoop.tar.gz"
|
|||
|
|
err = runCmd(cmd)
|
|||
|
|
if err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func downloadFileFromHttp(url string, output string) (string, error) {
|
|||
|
|
var localPath = output
|
|||
|
|
var cmd = "curl -sSLf " + url + " -o " + localPath
|
|||
|
|
var err = runCmd(cmd)
|
|||
|
|
if err != nil {
|
|||
|
|
return "", err
|
|||
|
|
}
|
|||
|
|
return localPath, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func downloadFileFromHadoop(jobConf *conf.JobConfig, hadoopHome string, javaHome string) (string, error) {
|
|||
|
|
t := time.Now().AddDate(0, 0, -1)
|
|||
|
|
day := t.Format("20060102")
|
|||
|
|
localPath := fmt.Sprintf(jobConf.LocalTmpFile, day)
|
|||
|
|
if ioutil2.FileExists(localPath) {
|
|||
|
|
return localPath, nil
|
|||
|
|
}
|
|||
|
|
remotePath := fmt.Sprintf(jobConf.HadoopFile, day)
|
|||
|
|
cmd := fmt.Sprintf("export JAVA_HOME=%s; %s/bin/hdfs dfs -get %s %s",
|
|||
|
|
javaHome, hadoopHome, remotePath, localPath)
|
|||
|
|
err := runCmd(cmd)
|
|||
|
|
return localPath, err
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// runCmd runs the cmd & print output (both stdout & stderr)
|
|||
|
|
func runCmd(cmd string) (err error) {
|
|||
|
|
fmt.Printf("CMD: %s \n", cmd)
|
|||
|
|
out, err := exec.Command("/bin/bash", "-c", cmd).CombinedOutput()
|
|||
|
|
log.Info(string(out))
|
|||
|
|
if err != nil {
|
|||
|
|
err = errors.Wrap(err, string(out))
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|