package crawler import ( "net/url" "github.com/satori/go.uuid" "gopkg.in/mgo.v2/bson" ) // URL is our reimplementation of url.URL to support marshalling type URL struct{ *url.URL } // UnmarshalText implements TextUnmarshaler (JSON, too) func (u *URL) UnmarshalText(text []byte) error { u2, err := url.Parse(string(text)) u.URL = u2 return err } // MarshalText implements TextMarshaler (JSON, too) func (u *URL) MarshalText() ([]byte, error) { return []byte(u.String()), nil } // A JobSubmission is what the Frontend will send to the crawler. // // The "real" thing to do is Job. Other fields are metadata, useful for the response type JobSubmission struct { Job bson.M // what to do ResponseRequested bool // shall the crawler send a Response to the Frontend? RequestID uuid.UUID `json:"requestID"` // this id is mainly intended to be passed back in the CrawlerResponse PingbackURL URL // when the request has been fulfilled, this URL must be visited with a CrawlerResponse } // A JobCompletionNotice is what the Crawler should send to the frontend after the Job has been successfully processed. type JobCompletionNotice struct { RequestID uuid.UUID // the one we received in the Request Error error // was everything fine? Other bson.M // a crawler might communicate something to the frontend }