优化日志库-新增Buff数量,减少多协程并发写锁等待

This commit is contained in:
boyce
2021-07-01 18:24:25 +08:00
parent a0594cb945
commit 17601f1831
3 changed files with 221 additions and 184 deletions

View File

@@ -1,17 +1,29 @@
package log // import "go.uber.org/zap/buffer"
import "strconv"
import (
"strconv"
"sync"
)
const _size = 9216
type Buffer struct {
bs []byte
mu sync.Mutex // ensures atomic writes; protects the following fields
}
func (buff *Buffer) Init(){
buff.bs = make([]byte,_size)
}
func (buff *Buffer) Locker() {
buff.mu.Lock()
}
func (buff *Buffer) UnLocker() {
buff.mu.Unlock()
}
// AppendByte writes a single byte to the Buffer.
func (b *Buffer) AppendByte(v byte) {
b.bs = append(b.bs, v)

View File

@@ -13,6 +13,7 @@ import (
"runtime/debug"
"strings"
"sync"
"sync/atomic"
"time"
)
@@ -46,12 +47,14 @@ type Logger struct {
fileDay int
level int
flag int
buf Buffer
buf []Buffer
outFile io.Writer // destination for output
outConsole io.Writer //os.Stdout
mu sync.Mutex // ensures atomic writes; protects the following fields
buffIndex uint32
buffNum uint32
}
func (logger *Logger) GenDayFile(now *time.Time) error {
@@ -84,7 +87,7 @@ func (logger *Logger) GenDayFile(now *time.Time) error {
return nil
}
func New(strLevel string, pathName string, filePre string, flag int) (*Logger, error) {
func New(strLevel string, pathName string, filePre string, flag int,buffNum uint32) (*Logger, error) {
// level
var level int
switch strings.ToLower(strLevel) {
@@ -110,7 +113,13 @@ func New(strLevel string, pathName string, filePre string, flag int) (*Logger, e
logger.filePath = pathName
logger.filepre = filePre
logger.flag = flag
logger.buf.Init()
logger.buf = make([]Buffer,buffNum)
logger.buffNum = buffNum
for i:=uint32(0);i<buffNum;i++{
logger.buf[i].Init()
}
now := time.Now()
err := logger.GenDayFile(&now)
if err != nil {
@@ -120,6 +129,11 @@ func New(strLevel string, pathName string, filePre string, flag int) (*Logger, e
return logger, nil
}
func (logger *Logger) nextBuff() *Buffer{
return &logger.buf[atomic.AddUint32(&logger.buffIndex,1)%logger.buffNum]
}
// It's dangerous to call the method on logging
func (logger *Logger) Close() {
if logger.outFile != nil {
@@ -134,21 +148,27 @@ func (logger *Logger) doPrintf(level int, printLevel string, format string, a ..
}
now := time.Now()
buf := logger.nextBuff()
buf.Locker()
buf.Reset()
logger.formatHeader(buf,3,&now)
buf.AppendString(printLevel)
buf.AppendString(fmt.Sprintf(format, a...))
buf.AppendByte('\n')
logger.mu.Lock()
logger.GenDayFile(&now)
logger.buf.Reset()
logger.formatHeader(3,&now)
logger.buf.AppendString(printLevel)
logger.buf.AppendString(fmt.Sprintf(format, a...))
logger.buf.AppendByte('\n')
if logger.outFile!= nil {
logger.outFile.Write(logger.buf.Bytes())
logger.outFile.Write(buf.Bytes())
}
if logger.outConsole!= nil {
logger.outConsole.Write(logger.buf.Bytes())
logger.outConsole.Write(buf.Bytes())
}
logger.mu.Unlock()
buf.UnLocker()
if level == fatalLevel {
os.Exit(1)
}
@@ -160,356 +180,361 @@ func (logger *Logger) doSPrintf(level int, printLevel string, a []interface{}) {
return
}
now := time.Now()
logger.mu.Lock()
logger.GenDayFile(&now)
logger.buf.Reset()
logger.formatHeader(3,&now)
logger.buf.AppendString(printLevel)
buf := logger.nextBuff()
buf.Locker()
buf.Reset()
logger.formatHeader(buf,3,&now)
buf.AppendString(printLevel)
for _,s := range a {
switch s.(type) {
//case error:
// logger.buf.AppendString(s.(error).Error())
case []string:
strSlice := s.([]string)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,str := range strSlice {
logger.buf.AppendString(str)
logger.buf.AppendByte(',')
buf.AppendString(str)
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case int:
logger.buf.AppendInt(int64(s.(int)))
buf.AppendInt(int64(s.(int)))
case []int:
intSlice := s.([]int)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendInt(int64(v))
logger.buf.AppendByte(',')
buf.AppendInt(int64(v))
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case int8:
logger.buf.AppendInt(int64(s.(int8)))
buf.AppendInt(int64(s.(int8)))
case []int8:
intSlice := s.([]int8)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendInt(int64(v))
logger.buf.AppendByte(',')
buf.AppendInt(int64(v))
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case int16:
logger.buf.AppendInt(int64(s.(int16)))
buf.AppendInt(int64(s.(int16)))
case []int16:
intSlice := s.([]int16)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendInt(int64(v))
logger.buf.AppendByte(',')
buf.AppendInt(int64(v))
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case int32:
logger.buf.AppendInt(int64(s.(int32)))
buf.AppendInt(int64(s.(int32)))
case []int32:
intSlice := s.([]int32)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendInt(int64(v))
logger.buf.AppendByte(',')
buf.AppendInt(int64(v))
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case int64:
logger.buf.AppendInt(s.(int64))
buf.AppendInt(s.(int64))
case []int64:
intSlice := s.([]int64)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendInt(v)
logger.buf.AppendByte(',')
buf.AppendInt(v)
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case uint:
logger.buf.AppendUint(uint64(s.(uint)))
buf.AppendUint(uint64(s.(uint)))
case []uint:
intSlice := s.([]uint)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendUint(uint64(v))
logger.buf.AppendByte(',')
buf.AppendUint(uint64(v))
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case uint8:
logger.buf.AppendUint(uint64(s.(uint8)))
buf.AppendUint(uint64(s.(uint8)))
case []uint8:
intSlice := s.([]uint8)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendUint(uint64(v))
logger.buf.AppendByte(',')
buf.AppendUint(uint64(v))
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case uint16:
logger.buf.AppendUint(uint64(s.(uint16)))
buf.AppendUint(uint64(s.(uint16)))
case []uint16:
intSlice := s.([]uint16)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendUint(uint64(v))
logger.buf.AppendByte(',')
buf.AppendUint(uint64(v))
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case uint32:
logger.buf.AppendUint(uint64(s.(uint32)))
buf.AppendUint(uint64(s.(uint32)))
case []uint32:
intSlice := s.([]uint32)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendUint(uint64(v))
logger.buf.AppendByte(',')
buf.AppendUint(uint64(v))
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case uint64:
logger.buf.AppendUint(s.(uint64))
buf.AppendUint(s.(uint64))
case []uint64:
intSlice := s.([]uint64)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendUint(v)
logger.buf.AppendByte(',')
buf.AppendUint(v)
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case float32:
logger.buf.AppendFloat(float64(s.(float32)),32)
buf.AppendFloat(float64(s.(float32)),32)
case []float32:
intSlice := s.([]float32)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendFloat(float64(v),32)
logger.buf.AppendByte(',')
buf.AppendFloat(float64(v),32)
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case float64:
logger.buf.AppendFloat(s.(float64),64)
buf.AppendFloat(s.(float64),64)
case []float64:
intSlice := s.([]float64)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendFloat(v,64)
logger.buf.AppendByte(',')
buf.AppendFloat(v,64)
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case bool:
logger.buf.AppendBool(s.(bool))
buf.AppendBool(s.(bool))
case []bool:
intSlice := s.([]bool)
logger.buf.AppendByte('[')
buf.AppendByte('[')
for _,v := range intSlice {
logger.buf.AppendBool(v)
logger.buf.AppendByte(',')
buf.AppendBool(v)
buf.AppendByte(',')
}
lastIdx := logger.buf.Len()-1
if logger.buf.Bytes()[lastIdx] == ',' {
logger.buf.Bytes()[lastIdx] = ']'
lastIdx := buf.Len()-1
if buf.Bytes()[lastIdx] == ',' {
buf.Bytes()[lastIdx] = ']'
}else{
logger.buf.AppendByte(']')
buf.AppendByte(']')
}
case string:
logger.buf.AppendString(s.(string))
buf.AppendString(s.(string))
case *int:
val := s.(*int)
if val != nil {
logger.buf.AppendInt(int64(*val))
buf.AppendInt(int64(*val))
}else{
logger.buf.AppendString("nil<*int>")
buf.AppendString("nil<*int>")
}
case *int8:
val := s.(*int8)
if val != nil {
logger.buf.AppendInt(int64(*val))
buf.AppendInt(int64(*val))
}else{
logger.buf.AppendString("nil<*int8>")
buf.AppendString("nil<*int8>")
}
case *int16:
val := s.(*int16)
if val != nil {
logger.buf.AppendInt(int64(*val))
buf.AppendInt(int64(*val))
}else{
logger.buf.AppendString("nil<*int16>")
buf.AppendString("nil<*int16>")
}
case *int32:
val := s.(*int32)
if val != nil {
logger.buf.AppendInt(int64(*val))
buf.AppendInt(int64(*val))
}else{
logger.buf.AppendString("nil<*int32>")
buf.AppendString("nil<*int32>")
}
case *int64:
val := s.(*int64)
if val != nil {
logger.buf.AppendInt(int64(*val))
buf.AppendInt(int64(*val))
}else{
logger.buf.AppendString("nil<*int64>")
buf.AppendString("nil<*int64>")
}
case *uint:
val := s.(*uint)
if val != nil {
logger.buf.AppendUint(uint64(*val))
buf.AppendUint(uint64(*val))
}else{
logger.buf.AppendString("nil<*uint>")
buf.AppendString("nil<*uint>")
}
case *uint8:
val := s.(*uint8)
if val != nil {
logger.buf.AppendUint(uint64(*val))
buf.AppendUint(uint64(*val))
}else{
logger.buf.AppendString("nil<*uint8>")
buf.AppendString("nil<*uint8>")
}
case *uint16:
val := s.(*uint16)
if val != nil {
logger.buf.AppendUint(uint64(*val))
buf.AppendUint(uint64(*val))
}else{
logger.buf.AppendString("nil<*uint16>")
buf.AppendString("nil<*uint16>")
}
case *uint32:
val := s.(*uint32)
if val != nil {
logger.buf.AppendUint(uint64(*val))
buf.AppendUint(uint64(*val))
}else{
logger.buf.AppendString("nil<*uint32>")
buf.AppendString("nil<*uint32>")
}
case *uint64:
val := s.(*uint64)
if val != nil {
logger.buf.AppendUint(uint64(*val))
buf.AppendUint(uint64(*val))
}else{
logger.buf.AppendString("nil<*uint64>")
buf.AppendString("nil<*uint64>")
}
case *float32:
val := s.(*float32)
if val != nil {
logger.buf.AppendFloat(float64(*val),32)
buf.AppendFloat(float64(*val),32)
}else{
logger.buf.AppendString("nil<*float32>")
buf.AppendString("nil<*float32>")
}
case *float64:
val := s.(*float32)
if val != nil {
logger.buf.AppendFloat(float64(*val),64)
buf.AppendFloat(float64(*val),64)
}else{
logger.buf.AppendString("nil<*float64>")
buf.AppendString("nil<*float64>")
}
case *bool:
val := s.(*bool)
if val != nil {
logger.buf.AppendBool(*val)
buf.AppendBool(*val)
}else{
logger.buf.AppendString("nil<*bool>")
buf.AppendString("nil<*bool>")
}
case *string:
val := s.(*string)
if val != nil {
logger.buf.AppendString(*val)
buf.AppendString(*val)
}else{
logger.buf.AppendString("nil<*string>")
buf.AppendString("nil<*string>")
}
//case []byte:
// logger.buf.AppendBytes(s.([]byte))
default:
//b,err := json.MarshalToString(s)
//if err != nil {
logger.buf.AppendString("<unknown type>")
buf.AppendString("<unknown type>")
//}else{
//logger.buf.AppendBytes(b)
//}
}
}
logger.buf.AppendByte('\n')
buf.AppendByte('\n')
logger.mu.Lock()
logger.GenDayFile(&now)
if logger.outFile!= nil {
logger.outFile.Write(logger.buf.Bytes())
logger.outFile.Write(buf.Bytes())
}
if logger.outConsole!= nil {
logger.outConsole.Write(logger.buf.Bytes())
logger.outConsole.Write(buf.Bytes())
}
logger.mu.Unlock()
buf.UnLocker()
if level == fatalLevel {
os.Exit(1)
}
@@ -539,7 +564,7 @@ func (logger *Logger) Fatal(format string, a ...interface{}) {
logger.doPrintf(fatalLevel, printFatalLevel, format, a...)
}
var gLogger, _ = New("debug", "", "", log.LstdFlags|log.Lshortfile)
var gLogger, _ = New("debug", "", "", log.LstdFlags|log.Lshortfile,1)
// It's dangerous to call the method on logging
func Export(logger *Logger) {
@@ -603,7 +628,7 @@ func SFatal(a ...interface{}) {
}
const timeFlag = syslog.Ldate|syslog.Ltime|syslog.Lmicroseconds
func (logger *Logger) formatHeader(calldepth int,t *time.Time) {
func (logger *Logger) formatHeader(buf *Buffer,calldepth int,t *time.Time) {
var file string
var line int
if logger.flag&(syslog.Lshortfile|syslog.Llongfile) != 0 {
@@ -617,33 +642,33 @@ func (logger *Logger) formatHeader(calldepth int,t *time.Time) {
}
if logger.flag&syslog.Lmsgprefix != 0 {
logger.buf.AppendString(logger.filepre)
buf.AppendString(logger.filepre)
}
if logger.flag&timeFlag != 0 {
if logger.flag&syslog.Ldate != 0 {
year, month, day := t.Date()
logger.buf.AppendInt(int64(year))
logger.buf.AppendByte('/')
logger.buf.AppendInt(int64(month))
logger.buf.AppendByte('/')
logger.buf.AppendInt(int64(day))
logger.buf.AppendByte(' ')
buf.AppendInt(int64(year))
buf.AppendByte('/')
buf.AppendInt(int64(month))
buf.AppendByte('/')
buf.AppendInt(int64(day))
buf.AppendByte(' ')
}
if logger.flag&(syslog.Ltime|syslog.Lmicroseconds) != 0 {
hour, min, sec := t.Clock()
logger.buf.AppendInt(int64(hour))
logger.buf.AppendByte(':')
logger.buf.AppendInt(int64(min))
logger.buf.AppendByte(':')
buf.AppendInt(int64(hour))
buf.AppendByte(':')
buf.AppendInt(int64(min))
buf.AppendByte(':')
logger.buf.AppendInt(int64(sec))
buf.AppendInt(int64(sec))
if logger.flag&syslog.Lmicroseconds != 0 {
logger.buf.AppendByte('.')
logger.buf.AppendInt(int64(t.Nanosecond()/1e3))
buf.AppendByte('.')
buf.AppendInt(int64(t.Nanosecond()/1e3))
}
logger.buf.AppendByte(' ')
buf.AppendByte(' ')
}
}
if logger.flag&(syslog.Lshortfile|syslog.Llongfile) != 0 {
@@ -657,13 +682,13 @@ func (logger *Logger) formatHeader(calldepth int,t *time.Time) {
}
file = short
}
logger.buf.AppendString(file)
logger.buf.AppendByte(':')
logger.buf.AppendInt(int64(line))
logger.buf.AppendString(": ")
buf.AppendString(file)
buf.AppendByte(':')
buf.AppendInt(int64(line))
buf.AppendString(": ")
}
if logger.flag&syslog.Lmsgprefix != 0 {
logger.buf.AppendString(logger.filepre)
buf.AppendString(logger.filepre)
}
}

View File

@@ -163,7 +163,7 @@ func initLog() error{
localnodeinfo := cluster.GetCluster().GetLocalNodeInfo()
filepre := fmt.Sprintf("%s_%d_", localnodeinfo.NodeName, localnodeinfo.NodeId)
logger,err := log.New(logLevel,logPath,filepre,slog.LstdFlags|slog.Lshortfile)
logger,err := log.New(logLevel,logPath,filepre,slog.LstdFlags|slog.Lshortfile,10)
if err != nil {
fmt.Printf("cannot create log file!\n")
return err
@@ -287,7 +287,7 @@ func GetConfigDir() string {
}
func SetSysLog(strLevel string, pathname string, flag int){
logs,_:= log.New(strLevel,pathname, "", flag)
logs,_:= log.New(strLevel,pathname, "", flag,10)
log.Export(logs)
}