40 lines
1.3 KiB
Go
40 lines
1.3 KiB
Go
package crawler
|
|
|
|
import (
|
|
"net/url"
|
|
|
|
"github.com/satori/go.uuid"
|
|
"gopkg.in/mgo.v2/bson"
|
|
)
|
|
|
|
// URL is our reimplementation of url.URL to support marshalling
|
|
type URL struct{ *url.URL }
|
|
|
|
// UnmarshalText implements TextUnmarshaler (JSON, too)
|
|
func (u *URL) UnmarshalText(text []byte) error {
|
|
u2, err := url.Parse(string(text))
|
|
u.URL = u2
|
|
return err
|
|
}
|
|
|
|
// MarshalText implements TextMarshaler (JSON, too)
|
|
func (u *URL) MarshalText() ([]byte, error) {
|
|
return []byte(u.String()), nil
|
|
}
|
|
|
|
// A JobSubmission is what the Frontend will send to the crawler.
|
|
//
|
|
// The "real" thing to do is Job. Other fields are metadata, useful for the response
|
|
type JobSubmission struct {
|
|
Job bson.M // what to do
|
|
ResponseRequested bool // shall the crawler send a Response to the Frontend?
|
|
RequestID uuid.UUID `json:"requestID"` // this id is mainly intended to be passed back in the CrawlerResponse
|
|
PingbackURL URL // when the request has been fulfilled, this URL must be visited with a CrawlerResponse
|
|
}
|
|
|
|
// A JobCompletionNotice is what the Crawler should send to the frontend after the Job has been successfully processed.
|
|
type JobCompletionNotice struct {
|
|
RequestID uuid.UUID // the one we received in the Request
|
|
Error error // was everything fine?
|
|
Other bson.M // a crawler might communicate something to the frontend
|
|
}
|