123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230 |
- package main
- import (
- "bufio"
- "log"
- "os"
- "strings"
- "sync"
- "time"
- )
- const (
- Good Class = "GOOD"
- Bad Class = "BAD"
- )
- //ByControlPlane contains all the channels we need.
- type ByControlPlane struct {
- BadTokens chan string
- GoodTokens chan string
- StatsTokens chan string
- }
- type safeClassifier struct {
- bayez *Classifier
- busy sync.Mutex
- }
- //ControPlane is the variabile
- var ControPlane ByControlPlane
- //ByClassifier is the structure containing our Pseudo-Bayes classifier.
- type ByClassifier struct {
- STATS sync.Map
- Learning safeClassifier
- Working safeClassifier
- Generation int64
- }
- //AddStats adds the statistics after proper blocking.
- func (c *ByClassifier) AddStats(action string) {
- var one int64 = 1
- if v, ok := c.STATS.Load(action); ok {
- c.STATS.Store(action, v.(int64)+1)
- } else {
- c.STATS.Store(action, one)
- }
- }
- //IsBAD inserts a bad key in the right place.
- func (c *ByClassifier) IsBAD(key string) {
- k := strings.Fields(key)
- log.Println("BAD Received", k)
- c.Learning.busy.Lock()
- defer c.Learning.busy.Unlock()
- c.Learning.bayez.Learn(k, Bad)
- log.Println("BAD Learned", k)
- }
- //IsGOOD inserts the key in the right place.
- func (c *ByClassifier) IsGOOD(key string) {
- k := strings.Fields(key)
- log.Println("GOOD Received", k)
- c.Learning.busy.Lock()
- defer c.Learning.busy.Unlock()
- c.Learning.bayez.Learn(k, Good)
- log.Println("GOOD Learned", k)
- }
- //Posterior calculates Shannon based entropy using bad and good as different distributions
- func (c *ByClassifier) Posterior(hdr string) map[string]float64 {
- tokens := sanitizeHeaders(hdr)
- ff := make(map[string]float64)
- if c.Generation == 0 {
- ff["BAD"] = 0.5
- ff["GOOD"] = 0.5
- return ff
- }
- log.Println("Posterior locking the Working Bayesian")
- c.Working.busy.Lock()
- defer c.Working.busy.Unlock()
- log.Println("Going to calculate the Scores")
- scores, _, _, err := c.Working.bayez.SafeProbScores(strings.Fields(tokens))
- log.Println("Scores calculated")
- if err == ErrUnderflow {
- ff["BAD"] = 0.5
- ff["GOOD"] = 0.5
- return ff
- }
- ff["GOOD"] = scores[0]
- ff["BAD"] = scores[1]
- return ff
- }
- func (c *ByClassifier) enroll() {
- ControPlane.BadTokens = make(chan string, 2048)
- ControPlane.GoodTokens = make(chan string, 2048)
- ControPlane.StatsTokens = make(chan string, 2048)
- c.Generation = 0
- c.Learning.bayez = NewClassifierTfIdf(Good, Bad)
- c.Working.bayez = NewClassifierTfIdf(Good, Bad)
- c.readInitList("blacklist.txt", "BAD")
- c.readInitList("whitelist.txt", "GOOD")
- go c.readBadTokens()
- go c.readGoodTokens()
- go c.readStatsTokens()
- go c.updateLearners()
- log.Println("Classifier populated...")
- }
- func (c *ByClassifier) readBadTokens() {
- log.Println("Start reading BAD tokens")
- for token := range ControPlane.BadTokens {
- log.Println("Received BAD Token: ", token)
- c.IsBAD(token)
- }
- }
- func (c *ByClassifier) readGoodTokens() {
- log.Println("Start reading GOOD tokens")
- for token := range ControPlane.GoodTokens {
- log.Println("Received GOOD Token: ", token)
- c.IsGOOD(token)
- }
- }
- func (c *ByClassifier) readStatsTokens() {
- log.Println("Start reading STATS tokens")
- for token := range ControPlane.StatsTokens {
- c.AddStats(token)
- }
- }
- func (c *ByClassifier) readInitList(filePath, class string) {
- inFile, err := os.Open(filePath)
- if err != nil {
- log.Println(err.Error() + `: ` + filePath)
- return
- }
- defer inFile.Close()
- scanner := bufio.NewScanner(inFile)
- for scanner.Scan() {
- if len(scanner.Text()) > 3 {
- switch class {
- case "BAD":
- log.Println("Loading into Blacklist: ", scanner.Text()) // the line
- c.IsBAD(scanner.Text())
- case "GOOD":
- log.Println("Loading into Whitelist: ", scanner.Text()) // the line
- c.IsGOOD(scanner.Text())
- }
- }
- }
- }
- func (c *ByClassifier) updateLearners() {
- log.Println("Bayes Updater Start...")
- ticker := time.NewTicker(10 * time.Second)
- for ; true; <-ticker.C {
- var currentGen int64
- log.Println("Maturity is:", Maturity)
- log.Println("Seniority is:", ProxyFlow.seniority)
- if Maturity > 0 {
- currentGen = ProxyFlow.seniority / Maturity
- } else {
- currentGen = 0
- }
- log.Println("Current Generation is: ", currentGen)
- log.Println("Working Generation is: ", c.Generation)
- if currentGen > c.Generation {
- c.Learning.busy.Lock()
- c.Working.busy.Lock()
- c.Working.bayez = c.Learning.bayez
- c.Working.bayez.ConvertTermsFreqToTfIdf()
- c.Learning.bayez = NewClassifierTfIdf(Good, Bad)
- c.Generation = currentGen
- log.Println("Generation Updated to: ", c.Generation)
- c.Learning.busy.Unlock()
- c.Working.busy.Unlock()
- }
- }
- }
|