// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
/*
Package log provides the means of querying an application's logs from
within an App Engine application.
Example:
c := appengine.NewContext(r)
query := &log.Query{
AppLogs: true,
Versions: []string{"1"},
}
for results := query.Run(c); ; {
record, err := results.Next()
if err == log.Done {
log.Infof(c, "Done processing results")
break
}
if err != nil {
log.Errorf(c, "Failed to retrieve next log: %v", err)
break
}
log.Infof(c, "Saw record %v", record)
}
*/
package log // import "google.golang.org/appengine/log"
import (
"errors"
"fmt"
"strings"
"time"
"github.com/golang/protobuf/proto"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/internal"
pb "google.golang.org/appengine/internal/log"
)
// Query defines a logs query.
type Query struct {
// Start time specifies the earliest log to return (inclusive).
StartTime time.Time
// End time specifies the latest log to return (exclusive).
EndTime time.Time
// Offset specifies a position within the log stream to resume reading from,
// and should come from a previously returned Record's field of the same name.
Offset []byte
// Incomplete controls whether active (incomplete) requests should be included.
Incomplete bool
// AppLogs indicates if application-level logs should be included.
AppLogs bool
// ApplyMinLevel indicates if MinLevel should be used to filter results.
ApplyMinLevel bool
// If ApplyMinLevel is true, only logs for requests with at least one
// application log of MinLevel or higher will be returned.
MinLevel int
// Versions is the major version IDs whose logs should be retrieved.
// Logs for specific modules can be retrieved by the specifying versions
// in the form "module:version"; the default module is used if no module
// is specified.
Versions []string
// A list of requests to search for instead of a time-based scan. Cannot be
// combined with filtering options such as StartTime, EndTime, Offset,
// Incomplete, ApplyMinLevel, or Versions.
RequestIDs []string
}
// AppLog represents a single application-level log.
type AppLog struct {
Time time.Time
Level int
Message string
}
// Record contains all the information for a single web request.
type Record struct {
AppID string
ModuleID string
VersionID string
RequestID []byte
IP string
Nickname string
AppEngineRelease string
// The time when this request started.
StartTime time.Time
// The time when this request finished.
EndTime time.Time
// Opaque cursor into the result stream.
Offset []byte
// The time required to process the request.
Latency time.Duration
MCycles int64
Method string
Resource string
HTTPVersion string
Status int32
// The size of the request sent back to the client, in bytes.
ResponseSize int64
Referrer string
UserAgent string
URLMapEntry string
Combined string
Host string
// The estimated cost of this request, in dollars.
Cost float64
TaskQueueName string
TaskName string
WasLoadingRequest bool
PendingTime time.Duration
Finished bool
AppLogs []AppLog
// Mostly-unique identifier for the instance that handled the request if available.
InstanceID string
}
// Result represents the result of a query.
type Result struct {
logs []*Record
context context.Context
request *pb.LogReadRequest
resultsSeen bool
err error
}
// Next returns the next log record,
func (qr *Result) Next() (*Record, error) {
if qr.err != nil {
return nil, qr.err
}
if len(qr.logs) > 0 {
lr := qr.logs[0]
qr.logs = qr.logs[1:]
return lr, nil
}
if qr.request.Offset == nil && qr.resultsSeen {
return nil, Done
}
if err := qr.run(); err != nil {
// Errors here may be retried, so don't store the error.
return nil, err
}
return qr.Next()
}
// Done is returned when a query iteration has completed.
var Done = errors.New("log: query has no more results")
// protoToAppLogs takes as input an array of pointers to LogLines, the internal
// Protocol Buffer representation of a single application-level log,
// and converts it to an array of AppLogs, the external representation
// of an application-level log.
func protoToAppLogs(logLines []*pb.LogLine) []AppLog {
appLogs := make([]AppLog, len(logLines))
for i, line := range logLines {
appLogs[i] = AppLog{
Time: time.Unix(0, *line.Time*1e3),
Level: int(*line.Level),
Message: *line.LogMessage,
}
}
return appLogs
}
// protoToRecord converts a RequestLog, the internal Protocol Buffer
// representation of a single request-level log, to a Record, its
// corresponding external representation.
func protoToRecord(rl *pb.RequestLog) *Record {
offset, err := proto.Marshal(rl.Offset)
if err != nil {
offset = nil
}
return &Record{
AppID: *rl.AppId,
ModuleID: rl.GetModuleId(),
VersionID: *rl.VersionId,
RequestID: rl.RequestId,
Offset: offset,
IP: *rl.Ip,
Nickname: rl.GetNickname(),
AppEngineRelease: string(rl.GetAppEngineRelease()),
StartTime: time.Unix(0, *rl.StartTime*1e3),
EndTime: time.Unix(0, *rl.EndTime*1e3),
Latency: time.Duration(*rl.Latency) * time.Microsecond,
MCycles: *rl.Mcycles,
Method: *rl.Method,
Resource: *rl.Resource,
HTTPVersion: *rl.HttpVersion,
Status: *rl.Status,
ResponseSize: *rl.ResponseSize,
Referrer: rl.GetReferrer(),
UserAgent: rl.GetUserAgent(),
URLMapEntry: *rl.UrlMapEntry,
Combined: *rl.Combined,
Host: rl.GetHost(),
Cost: rl.GetCost(),
TaskQueueName: rl.GetTaskQueueName(),
TaskName: rl.GetTaskName(),
WasLoadingRequest: rl.GetWasLoadingRequest(),
PendingTime: time.Duration(rl.GetPendingTime()) * time.Microsecond,
Finished: rl.GetFinished(),
AppLogs: protoToAppLogs(rl.Line),
InstanceID: string(rl.GetCloneKey()),
}
}
// Run starts a query for log records, which contain request and application
// level log information.
func (params *Query) Run(c context.Context) *Result {
req, err := makeRequest(params, internal.FullyQualifiedAppID(c), appengine.VersionID(c))
return &Result{
context: c,
request: req,
err: err,
}
}
func makeRequest(params *Query, appID, versionID string) (*pb.LogReadRequest, error) {
req := &pb.LogReadRequest{}
req.AppId = &appID
if !params.StartTime.IsZero() {
req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3)
}
if !params.EndTime.IsZero() {
req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3)
}
if len(params.Offset) > 0 {
var offset pb.LogOffset
if err := proto.Unmarshal(params.Offset, &offset); err != nil {
return nil, fmt.Errorf("bad Offset: %v", err)
}
req.Offset = &offset
}
if params.Incomplete {
req.IncludeIncomplete = ¶ms.Incomplete
}
if params.AppLogs {
req.IncludeAppLogs = ¶ms.AppLogs
}
if params.ApplyMinLevel {
req.MinimumLogLevel = proto.Int32(int32(params.MinLevel))
}
if params.Versions == nil {
// If no versions were specified, default to the default module at
// the major version being used by this module.
if i := strings.Index(versionID, "."); i >= 0 {
versionID = versionID[:i]
}
req.VersionId = []string{versionID}
} else {
req.ModuleVersion = make([]*pb.LogModuleVersion, 0, len(params.Versions))
for _, v := range params.Versions {
var m *string
if i := strings.Index(v, ":"); i >= 0 {
m, v = proto.String(v[:i]), v[i+1:]
}
req.ModuleVersion = append(req.ModuleVersion, &pb.LogModuleVersion{
ModuleId: m,
VersionId: proto.String(v),
})
}
}
if params.RequestIDs != nil {
ids := make([][]byte, len(params.RequestIDs))
for i, v := range params.RequestIDs {
ids[i] = []byte(v)
}
req.RequestId = ids
}
return req, nil
}
// run takes the query Result produced by a call to Run and updates it with
// more Records. The updated Result contains a new set of logs as well as an
// offset to where more logs can be found. We also convert the items in the
// response from their internal representations to external versions of the
// same structs.
func (r *Result) run() error {
res := &pb.LogReadResponse{}
if err := internal.Call(r.context, "logservice", "Read", r.request, res); err != nil {
return err
}
r.logs = make([]*Record, len(res.Log))
r.request.Offset = res.Offset
r.resultsSeen = true
for i, log := range res.Log {
r.logs[i] = protoToRecord(log)
}
return nil
}
func init() {
internal.RegisterErrorCodeMap("logservice", pb.LogServiceError_ErrorCode_name)
}