Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,61 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"hbase_face_test.go",
"hbase_login_log_test.go",
"row_key_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/passport/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
"//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"hbase_face.go",
"hbase_login_log.go",
"hbase_pwd.go",
"row_key.go",
],
importpath = "go-common/app/service/main/passport/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/passport/conf:go_default_library",
"//app/service/main/passport/model:go_default_library",
"//library/database/hbase.v2:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/tsuna/gohbase/filter:go_default_library",
"//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,50 @@
package dao
import (
"context"
"go-common/app/service/main/passport/conf"
"go-common/library/database/hbase.v2"
)
// Dao struct answer history of Dao
type Dao struct {
c *conf.Config
hbase *hbase.Client
loginLogHBase *hbase.Client
pwdLogHBase *hbase.Client
}
// New new a Dao and return.
func New(c *conf.Config) (d *Dao) {
var loginLogHBase *hbase.Client
if c.Switch.LoginLogHBase {
loginLogHBase = hbase.NewClient(c.HBase.LoginLog.Config)
}
d = &Dao{
c: c,
hbase: hbase.NewClient(c.HBase.FaceApply.Config),
loginLogHBase: loginLogHBase,
pwdLogHBase: hbase.NewClient(c.HBase.PwdLog.Config),
}
return
}
// Close close connections.
func (d *Dao) Close() {
if d.hbase != nil {
d.hbase.Close()
}
if d.loginLogHBase != nil {
d.loginLogHBase.Close()
}
if d.pwdLogHBase != nil {
d.pwdLogHBase.Close()
}
}
// Ping ping health.
func (d *Dao) Ping(c context.Context) (err error) {
return
}

View File

@@ -0,0 +1,34 @@
package dao
import (
"flag"
"os"
"testing"
"go-common/app/service/main/passport/conf"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.account.passport-service")
flag.Set("conf_token", "c643a3cf59902ee7907946f12bdba10c")
flag.Set("tree_id", "7812")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
m.Run()
os.Exit(0)
}

View File

@@ -0,0 +1,166 @@
package dao
import (
"bytes"
"context"
"io"
"strconv"
"time"
"go-common/app/service/main/passport/model"
"go-common/library/log"
"github.com/tsuna/gohbase/filter"
"github.com/tsuna/gohbase/hrpc"
)
const (
_tFace = "account:user_face"
_fFaceApply = "c"
_cMid = "mid"
_cApplyTime = "at"
_cModifyTime = "mt"
_cNewFace = "nf"
_cOldFace = "of"
_cOperator = "op"
_cStatus = "s"
_maxIDLen = 10
)
var (
_tFaceB = []byte(_tFace)
_cMidB = []byte(_cMid)
_fFaceApplyB = []byte(_fFaceApply)
_cApplyTimeB = []byte(_cApplyTime)
_cModifyTimeB = []byte(_cModifyTime)
_cNewFaceB = []byte(_cNewFace)
_cOldFaceB = []byte(_cOldFace)
_cOperatorB = []byte(_cOperator)
_cStatusB = []byte(_cStatus)
)
// FaceApplies get face applies from hbase.
func (d *Dao) FaceApplies(c context.Context, mid, from, to int64, status, operator string) (res []*model.FaceApply, err error) {
midStr := strconv.FormatInt(mid, 10)
if !checkIDLen(midStr) {
log.Error("midInt64: %d, midStr: %s, len(midStr): %d, exceed max length %d", mid, midStr, len(midStr), _maxIDLen)
return
}
ctx, cancel := context.WithTimeout(c, time.Duration(d.c.HBase.FaceApply.ReadTimeout))
defer cancel()
st := rowKeyFaceApplyMts(midStr, to, _uint32Max)
if len(st) == 0 {
return
}
ed := rowKeyFaceApplyMts(midStr, from, 0)
if len(st) == 0 {
return
}
fts := filter.NewList(filter.MustPassAll)
if status != "" {
statusFt := filter.NewSingleColumnValueFilter(_fFaceApplyB, _cStatusB, filter.Equal, filter.NewBinaryComparator(filter.NewByteArrayComparable([]byte(status))), true, true)
fts.AddFilters(statusFt)
}
if operator != "" {
operatorFt := filter.NewSingleColumnValueFilter(_fFaceApplyB, _cOperatorB, filter.Equal, filter.NewBinaryComparator(filter.NewByteArrayComparable([]byte(operator))), true, true)
fts.AddFilters(operatorFt)
}
var options []func(hrpc.Call) error
if len(fts.Filters) > 0 {
options = append(options, hrpc.Filters(fts))
}
var scaner hrpc.Scanner
scaner, err = d.hbase.ScanRange(ctx, _tFaceB, st, ed, options...)
if err != nil {
log.Error("hbase.ScanRange(%s, %s, %s) error(%v)", _tFaceB, st, ed, err)
return
}
var rs []*hrpc.Result
for {
var r *hrpc.Result
r, err = scaner.Next()
if err != nil {
if err == io.EOF {
// set err nil
err = nil
break
}
return
}
rs = append(rs, r)
}
if len(rs) == 0 {
return
}
res = make([]*model.FaceApply, 0, len(rs))
for _, r := range rs {
var u *model.FaceApply
if u, err = scanFaceRecord(r.Cells); err != nil {
return
}
if u != nil {
res = append(res, u)
}
}
return
}
// rowKeyFaceApplyMts get row key for face apply using this schema:
// mid string reverse with right fill 0 +
// (int64_max - mts) string cut last 10 digit +
// (unsigned_int32_max - id) string with left fill 0.
func rowKeyFaceApplyMts(midStr string, mts, id int64) []byte {
rMid := reverseID(midStr, 10)
if len(rMid) == 0 {
return nil
}
b := bytes.Buffer{}
b.WriteString(rMid)
rMTS := diffTs(mts)
b.WriteString(rMTS)
rID := diffID(id)
b.WriteString(rID)
return b.Bytes()
}
func scanFaceRecord(cells []*hrpc.Cell) (res *model.FaceApply, err error) {
if len(cells) == 0 {
return
}
res = new(model.FaceApply)
for _, cell := range cells {
if bytes.Equal(cell.Family, _fFaceApplyB) {
switch {
case bytes.Equal(cell.Qualifier, _cMidB):
if res.Mid, err = strconv.ParseInt(string(cell.Value), 10, 64); err != nil {
log.Error("failed to parse mid from cell, strconv.ParseInt(%s, 10, 64) error(%v)", cell.Value, err)
return
}
case bytes.Equal(cell.Qualifier, _cOldFaceB):
res.OldFace = string(cell.Value)
case bytes.Equal(cell.Qualifier, _cNewFaceB):
res.NewFace = string(cell.Value)
case bytes.Equal(cell.Qualifier, _cApplyTimeB):
if res.ApplyTime, err = strconv.ParseInt(string(cell.Value), 10, 64); err != nil {
log.Error("failed to parse apply_time from cell, strconv.ParseInt(%s, 10, 64) error(%v)", cell.Value, err)
return
}
case bytes.Equal(cell.Qualifier, _cStatusB):
res.Status = string(cell.Value)
case bytes.Equal(cell.Qualifier, _cOperatorB):
res.Operator = string(cell.Value)
case bytes.Equal(cell.Qualifier, _cModifyTimeB):
res.ModifyTime = string(cell.Value)
}
}
}
return
}

View File

@@ -0,0 +1,88 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
"github.com/tsuna/gohbase/hrpc"
)
func TestDaoFaceApplies(t *testing.T) {
var (
c = context.TODO()
mid = int64(100)
from, to = int64(0), int64(_uint32Max)
status = ""
operator = ""
)
convey.Convey("FaceApplies", t, func(ctx convey.C) {
_, err := d.FaceApplies(c, mid, from, to, status, operator)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
//ctx.So(res, convey.ShouldNotBeNil)
//
//lastTs := int64(0)
//lastID := int64(0)
//for _, v := range res {
// ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
// mt, err := time.ParseInLocation(_inputFormat, v.ModifyTime, _loc)
// ctx.So(err, convey.ShouldBeNil)
//
// id, err := strconv.ParseInt(v.Operator, 10, 64)
// ctx.So(err, convey.ShouldBeNil)
//
// // check ts seq
// ts := mt.Unix()
// if lastTs > 0 {
// ctx.So(lastTs, convey.ShouldBeGreaterThanOrEqualTo, ts)
// }
//
// // check id seq when ts equal
// if lastTs == ts && lastID > 0 {
// ctx.So(lastID, convey.ShouldBeGreaterThan, id)
// }
//
// lastTs = ts
// lastID = id
// })
//}
})
})
}
func TestDaorowKeyFaceApplyMts(t *testing.T) {
var (
midStr = "123"
mts = int64(0)
id = int64(0)
)
convey.Convey("rowKeyFaceApplyMts", t, func(ctx convey.C) {
p1 := rowKeyFaceApplyMts(midStr, mts, id)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestDaoscanFaceRecord(t *testing.T) {
var (
cells = []*hrpc.Cell{
{Family: []byte("c"), Qualifier: []byte("mid"), Value: []byte("123")},
{Family: []byte("c"), Qualifier: []byte("at"), Value: []byte("1530846373")},
{Family: []byte("c"), Qualifier: []byte("mt"), Value: []byte("1530846373")},
{Family: []byte("c"), Qualifier: []byte("nf"), Value: []byte("nf")},
{Family: []byte("c"), Qualifier: []byte("of"), Value: []byte("of")},
{Family: []byte("c"), Qualifier: []byte("op"), Value: []byte("op")},
{Family: []byte("c"), Qualifier: []byte("s"), Value: []byte("0")},
}
)
convey.Convey("scanFaceRecord", t, func(ctx convey.C) {
res, err := scanFaceRecord(cells)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}

View File

@@ -0,0 +1,137 @@
package dao
import (
"bytes"
"context"
"encoding/binary"
"io"
"strconv"
"time"
"go-common/app/service/main/passport/model"
"go-common/library/log"
"github.com/tsuna/gohbase/hrpc"
)
const (
_tLoginLog = "ugc:AsoLoginLog"
_fLoginLogInfo = "f"
_cLoginLogMid = "mid"
_cLoginLogTs = "ts"
_cLoginLogLoginIP = "ip"
_cLoginLogType = "t"
_cLoginLogServer = "s"
)
var (
_tLoginLogB = []byte(_tLoginLog)
_fLoginLogInfoB = []byte(_fLoginLogInfo)
_cLoginLogMidB = []byte(_cLoginLogMid)
_cLoginLogTsB = []byte(_cLoginLogTs)
_cLoginLogLoginIPB = []byte(_cLoginLogLoginIP)
_cLoginLogTypeB = []byte(_cLoginLogType)
_cLoginLogServerB = []byte(_cLoginLogServer)
)
// LoginLogs get last limit login logs.
func (d *Dao) LoginLogs(c context.Context, mid int64, limit int) (res []*model.LoginLog, err error) {
ctx, cancel := context.WithTimeout(c, time.Duration(d.c.HBase.LoginLog.ReadTimeout))
defer cancel()
st := rowKeyLoginLog(mid, _int64Max)
ed := rowKeyLoginLog(mid, 0)
var scaner hrpc.Scanner
scaner, err = d.loginLogHBase.ScanRange(ctx, _tLoginLogB, st, ed)
if err != nil {
log.Error("hbase.ScanRange(%s, %s, %s) error(%v)", _tLoginLogB, st, ed, err)
return
}
res = make([]*model.LoginLog, 0)
for ; limit > 0; limit-- {
var u *model.LoginLog
var r *hrpc.Result
r, err = scaner.Next()
if err != nil {
if err == io.EOF {
err = nil
break
}
return
}
if u, err = scanLoginLog(r.Cells); err != nil {
scaner.Close()
return
}
if u != nil {
res = append(res, u)
}
}
if err := scaner.Close(); err != nil {
log.Error("hbase.Scanner.Close error(%v)", err)
}
return
}
// rowKeyLoginLog generate row key of login log.
func rowKeyLoginLog(mid, ts int64) (res []byte) {
buf := bytes.Buffer{}
b := make([]byte, 8)
// reverse mid bytes
binary.BigEndian.PutUint64(b, uint64(mid))
reverse(b)
buf.Write(b)
// (int64_max - ts) bytes
binary.BigEndian.PutUint64(b, uint64(_int64Max-ts))
buf.Write(b)
res = buf.Bytes()
return
}
func reverse(b []byte) {
l := len(b)
for i := 0; i < l/2; i++ {
t := b[i]
b[i] = b[l-1-i]
b[l-1-i] = t
}
}
func scanLoginLog(cells []*hrpc.Cell) (res *model.LoginLog, err error) {
if len(cells) == 0 {
return
}
res = new(model.LoginLog)
for _, cell := range cells {
if bytes.Equal(cell.Family, _fLoginLogInfoB) {
switch {
case bytes.Equal(cell.Qualifier, _cLoginLogMidB):
if res.Mid, err = strconv.ParseInt(string(cell.Value), 10, 64); err != nil {
log.Error("failed to parse mid from cell, strconv.ParseInt(%s, 10, 64) error(%v)", cell.Value, err)
return
}
case bytes.Equal(cell.Qualifier, _cLoginLogTsB):
if res.Timestamp, err = strconv.ParseInt(string(cell.Value), 10, 64); err != nil {
log.Error("failed to parse timestamp from cell, strconv.ParseInt(%s, 10, 64) error(%v)", cell.Value, err)
return
}
case bytes.Equal(cell.Qualifier, _cLoginLogLoginIPB):
if res.LoginIP, err = strconv.ParseInt(string(cell.Value), 10, 64); err != nil {
log.Error("failed to parse loginip from cell, strconv.ParseInt(%s, 10, 64) error(%v)", cell.Value, err)
return
}
case bytes.Equal(cell.Qualifier, _cLoginLogTypeB):
if res.Type, err = strconv.ParseInt(string(cell.Value), 10, 64); err != nil {
log.Error("failed to parse type from cell, strconv.ParseInt(%s, 10, 64) error(%v)", cell.Value, err)
return
}
case bytes.Equal(cell.Qualifier, _cLoginLogServerB):
res.Server = string(cell.Value)
}
}
}
return
}

View File

@@ -0,0 +1,67 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
"github.com/tsuna/gohbase/hrpc"
)
func TestDaoLoginLogs(t *testing.T) {
var (
c = context.TODO()
mid = int64(123)
limit = int(1)
)
convey.Convey("LoginLogs", t, func(ctx convey.C) {
res, err := d.LoginLogs(c, mid, limit)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}
func TestDaorowKeyLoginLog(t *testing.T) {
var (
mid = int64(123)
ts = int64(1530846373)
)
convey.Convey("rowKeyLoginLog", t, func(ctx convey.C) {
res := rowKeyLoginLog(mid, ts)
ctx.Convey("Then res should not be nil.", func(ctx convey.C) {
ctx.So(res, convey.ShouldNotBeNil)
})
})
}
func TestDaoreverse(t *testing.T) {
var (
b = []byte("123")
)
convey.Convey("reverse", t, func(ctx convey.C) {
reverse(b)
ctx.Convey("No return values", func(ctx convey.C) {
})
})
}
func TestDaoscanLoginLog(t *testing.T) {
var (
cells = []*hrpc.Cell{
{Family: []byte("f"), Qualifier: []byte("mid"), Value: []byte("123")},
{Family: []byte("f"), Qualifier: []byte("ts"), Value: []byte("1530846373")},
{Family: []byte("f"), Qualifier: []byte("ip"), Value: []byte("2886731194")},
{Family: []byte("f"), Qualifier: []byte("t"), Value: []byte("0")},
{Family: []byte("f"), Qualifier: []byte("s"), Value: []byte("s")},
}
)
convey.Convey("scanLoginLog", t, func(ctx convey.C) {
res, err := scanLoginLog(cells)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}

View File

@@ -0,0 +1,103 @@
package dao
import (
"bytes"
"context"
"encoding/binary"
"io"
"time"
"go-common/app/service/main/passport/model"
"go-common/library/log"
"github.com/tsuna/gohbase/hrpc"
)
const (
_tPwdLog = "ugc:PwdLog"
_fPwdLog = "pwdlog"
_cPwdLogOldPwd = "old_pwd"
_cPwdLogOldSalt = "old_salt"
)
var (
_tPwdLogB = []byte(_tPwdLog)
_fPwdLogB = []byte(_fPwdLog)
_cPwdLogOldPwdB = []byte(_cPwdLogOldPwd)
_cPwdLogOldSaltB = []byte(_cPwdLogOldSalt)
)
// HistoryPwds get history pwd
func (d *Dao) HistoryPwds(c context.Context, mid int64) (res []*model.HistoryPwd, err error) {
ctx, cancel := context.WithTimeout(c, time.Duration(d.c.HBase.LoginLog.ReadTimeout))
defer cancel()
st := RowKeyPwdLog(mid, _int64Max)
ed := RowKeyPwdLog(mid, 0)
var scaner hrpc.Scanner
scaner, err = d.pwdLogHBase.ScanRange(ctx, _tPwdLogB, st, ed)
if err != nil {
log.Error("hbase.ScanRange(%s, %s, %s) error(%v)", _tPwdLogB, st, ed, err)
return
}
res = make([]*model.HistoryPwd, 0)
for {
var pwd *model.HistoryPwd
var r *hrpc.Result
r, err = scaner.Next()
if err != nil {
if err == io.EOF {
err = nil
break
}
return
}
if pwd, err = scanPwdLog(r.Cells); err != nil {
scaner.Close()
return
}
if pwd != nil {
res = append(res, pwd)
}
}
if err := scaner.Close(); err != nil {
log.Error("hbase.Scanner.Close error(%v)", err)
}
return
}
// RowKeyPwdLog generate row key of pwd log.
func RowKeyPwdLog(mid, ts int64) (res []byte) {
buf := bytes.Buffer{}
b := make([]byte, 8)
// reverse mid bytes
binary.BigEndian.PutUint64(b, uint64(mid))
reverse(b)
buf.Write(b)
// (int64_max - ts) bytes
binary.BigEndian.PutUint64(b, uint64(_int64Max-ts))
buf.Write(b)
res = buf.Bytes()
return
}
func scanPwdLog(cells []*hrpc.Cell) (res *model.HistoryPwd, err error) {
if len(cells) == 0 {
return
}
res = new(model.HistoryPwd)
for _, cell := range cells {
if bytes.Equal(cell.Family, _fPwdLogB) {
switch {
case bytes.Equal(cell.Qualifier, _cPwdLogOldPwdB):
res.OldPwd = string(cell.Value)
case bytes.Equal(cell.Qualifier, _cPwdLogOldSaltB):
res.OldSalt = string(cell.Value)
}
}
}
return
}

View File

@@ -0,0 +1,60 @@
package dao
import (
"strconv"
"strings"
"go-common/library/log"
)
const (
_int64Max = 0x7fffffffffffffff
_uint32Max = 0xffffffff
)
// reverseID reverse a digital number represented in string,
// if len(id) < len, fill 0 on the right of reverse id to make reverse id len 10,
// if len(id) > len, will return empty string.
func reverseID(id string, l int) string {
if len(id) > l {
log.Error("len(%s) is %d, greater than the given l %d", id, len(id), l)
return ""
}
// reverse id string
runes := []rune(id)
for from, to := 0, len(runes)-1; from < to; from, to = from+1, to-1 {
runes[from], runes[to] = runes[to], runes[from]
}
rid := string(runes)
if len(id) == l {
return rid
}
// fill with 0 on rid's right
rid += strings.Repeat("0", l-len(id))
return rid
}
func checkIDLen(id string) bool {
return len(id) <= _maxIDLen
}
// diffTs return the last 10 digit of (int64_max - ts).
func diffTs(ts int64) string {
i := _int64Max - ts
s := strconv.FormatInt(i, 10)
// during ts 0 - (int64 - now), cut the [9,19) part of s as result
return s[9:19]
}
// diffID return the (unsigned_int32_max - id) convert to string in base 10.
// if len of the string < 10, fill 0 on the left to make len(res) equal to 10.
func diffID(id int64) string {
i := _uint32Max - id
s := strconv.FormatInt(i, 10)
if len(s) == 10 {
return s
}
return strings.Repeat("0", 10-len(s)) + s
}

View File

@@ -0,0 +1,56 @@
package dao
import (
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoreverseID(t *testing.T) {
var (
id = "123"
l = int(3)
)
convey.Convey("reverseID", t, func(ctx convey.C) {
p1 := reverseID(id, l)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestDaocheckIDLen(t *testing.T) {
var (
id = "123"
)
convey.Convey("checkIDLen", t, func(ctx convey.C) {
p1 := checkIDLen(id)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestDaodiffTs(t *testing.T) {
var (
ts = int64(0)
)
convey.Convey("diffTs", t, func(ctx convey.C) {
p1 := diffTs(ts)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestDaodiffID(t *testing.T) {
var (
id = int64(0)
)
convey.Convey("diffID", t, func(ctx convey.C) {
p1 := diffID(id)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}