From 90c5d5f943faf0b3c530a4619f7bb3b30a00c974 Mon Sep 17 00:00:00 2001 From: Vitaly Pashkov Date: Mon, 21 Nov 2016 18:49:20 +0300 Subject: [PATCH] Initial project upload --- .gitignore | 24 +++++++++++++ README.md | 21 +++++++++++ common/sources.go | 23 ++++++++++++ common/targets.go | 23 ++++++++++++ common/utils.go | 52 +++++++++++++++++++++++++++ main.go | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++ sources/antizapret.go | 60 +++++++++++++++++++++++++++++++ sources/file.go | 54 ++++++++++++++++++++++++++++ sources/ipdeny.go | 55 +++++++++++++++++++++++++++++ sources/openbl.go | 52 +++++++++++++++++++++++++++ sources/test.go | 24 +++++++++++++ targets/file.go | 36 +++++++++++++++++++ targets/ipset.go | 46 ++++++++++++++++++++++++ targets/stdout.go | 24 +++++++++++++ 14 files changed, 592 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 common/sources.go create mode 100644 common/targets.go create mode 100644 common/utils.go create mode 100644 main.go create mode 100644 sources/antizapret.go create mode 100644 sources/file.go create mode 100644 sources/ipdeny.go create mode 100644 sources/openbl.go create mode 100644 sources/test.go create mode 100644 targets/file.go create mode 100644 targets/ipset.go create mode 100644 targets/stdout.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2661785 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c5d795e --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +# IPListget + +IPListget is a network list downloader. It can download a network list from various sources and apply it at different targets. For example, it can download a list from openbl.org and put it in ipset list. + +Below you will find the list of available sources and targets. + +Sources: + - antizapret - antizapret.info list of resources, blocked by Russian government + - openbl - openbl.org blacklist of internet abusers + - ipdeny - ipdeny.com country block lists + - file - reads IPs from a file (one per line) + - test - test source generating a list of 127.254.254.0/24 for test purposes + +Targets: + - ipset - inserts networks to ipset list + - file - writes networks to a file (one per line) + - stdout - prints a list to stdout + +More sources and targets will be added in the future. The following goals are: + - more sources of potentially dangerous and unwanted addresses, like tor proxys + - targets for different network devices, like routers and firewalls, especially Cisco, Juniper and Mikrotik diff --git a/common/sources.go b/common/sources.go new file mode 100644 index 0000000..13fb54a --- /dev/null +++ b/common/sources.go @@ -0,0 +1,23 @@ +package common + +import ( + "net" + "reflect" + "strings" +) + +var ( + Sources map[string]SourceModuleType //Available source modules storage +) + +type SourceModuleType interface { + Get(url string) ([]net.IPNet, error) +} + +func init() { + Sources = make(map[string]SourceModuleType) +} + +func RegisterSource(src SourceModuleType) { + Sources[strings.ToLower(reflect.ValueOf(src).Type().Name())] = src +} diff --git a/common/targets.go b/common/targets.go new file mode 100644 index 0000000..5980786 --- /dev/null +++ b/common/targets.go @@ -0,0 +1,23 @@ +package common + +import ( + "net" + "reflect" + "strings" +) + +var ( + Targets map[string]TargetModuleType //Available target modules storage +) + +type TargetModuleType interface { + Set(destination string, iplist []net.IPNet) error +} + +func init() { + Targets = make(map[string]TargetModuleType) +} + +func RegisterTarget(tgt TargetModuleType) { + Targets[strings.ToLower(reflect.ValueOf(tgt).Type().Name())] = tgt +} diff --git a/common/utils.go b/common/utils.go new file mode 100644 index 0000000..e33e82e --- /dev/null +++ b/common/utils.go @@ -0,0 +1,52 @@ +package common + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" +) + +func FetchURL(url string) ([]byte, error) { + resp, err := http.Get(url) + if resp != nil { + defer resp.Body.Close() + } + + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + return nil, errors.New(fmt.Sprintf("Request failed, http status: %v", resp.StatusCode)) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return body, nil +} + +func URLReader(url string) (io.ReadCloser, error) { + resp, err := http.Get(url) + + if resp == nil { + return nil, errors.New("Server returned empty response") + } + + if err != nil { + // Even on error response may be non-nil + resp.Body.Close() + return nil, err + } + + if resp.StatusCode != 200 { + resp.Body.Close() + return nil, errors.New(fmt.Sprintf("Request failed, http status: %v", resp.StatusCode)) + } + + return resp.Body, nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..d8ac626 --- /dev/null +++ b/main.go @@ -0,0 +1,98 @@ +// iplistget project main.go +// Production build: go build -ldflags="-w" -o iplistget_release.exe iplistget +package main + +import ( + "flag" + "fmt" + "iplistget/common" + _ "iplistget/sources" + _ "iplistget/targets" + "net" + "os" +) + +const ( + APP_NAME = "iplistget network lists operator" + VERSION = "0.1-dev" +) + +var ( + // Application flags + showVersion bool // Show version and exit + //sudo bool // Use sudo for privileged operations + listmodules bool // List available sources + source string // Module name to fetch the list + target string // Module name to supply this list to (iproute, ipset, file, stdout) + destination string // Optional parameter where to apply (routing table, ipset list, filename) + sourceurl string // Override the default source url +) + +func init() { + flag.BoolVar(&showVersion, "version", false, "Show version and exit") + flag.BoolVar(&listmodules, "modules", false, "List all available modules and exit") + flag.StringVar(&source, "source", "", "Use specified source module") + flag.StringVar(&sourceurl, "url", "", "Override the default URL for source module") + flag.StringVar(&target, "target", "", "Use specified target module") + flag.StringVar(&destination, "destination", "", "Override the default destination (table name, list name, filename)") + //flag.BoolVar(&sudo, "sudo", false, "Use sudo for privileged operations") +} + +func main() { + flag.Parse() + + if showVersion { + fmt.Printf("%s version %s\n", APP_NAME, VERSION) + return + } + + if listmodules { + lsmodules() + return + } + + if source == "" { + fmt.Println("No source module specified") + os.Exit(1) + } + + if target == "" { + fmt.Println("No destination module provided") + os.Exit(1) + } + + srcmod, srcok := common.Sources[source] + if srcok != true { + fmt.Printf("Error. No such source module `%v`!\n", source) + os.Exit(1) + } + + tgtmod, tgtok := common.Targets[target] + if tgtok != true { + fmt.Printf("Error. No such target module `%v`!\n", target) + os.Exit(1) + } + + var iplist []net.IPNet + iplist, geterr := srcmod.Get(sourceurl) + if geterr != nil { + fmt.Printf("Error. Source module failed. %v\n", geterr) + } + + if err := tgtmod.Set(destination, iplist); err != nil { + fmt.Printf("Error. Target module failed. %v\n", err) + } +} + +func lsmodules() { + fmt.Print("Available sources: ") + for src := range common.Sources { + fmt.Print(src, " ") + } + + fmt.Print("\nAvailable targets: ") + for tgt := range common.Targets { + fmt.Print(tgt, " ") + } + fmt.Println() +} diff --git a/sources/antizapret.go b/sources/antizapret.go new file mode 100644 index 0000000..ffb155e --- /dev/null +++ b/sources/antizapret.go @@ -0,0 +1,60 @@ +package sources + +import ( + "bufio" + "bytes" + "io" + "iplistget/common" + "net" + "strings" +) + +func init() { + common.RegisterSource(Antizapret{URL: "https://api.antizapret.info/group.php?data=ip"}) +} + +type Antizapret struct { + URL string +} + +func (s Antizapret) Get(url string) ([]net.IPNet, error) { + if url != "" { + s.URL = url + } + + data, err := common.FetchURL(s.URL) + if err != nil { + return nil, err + } + + ipmap := map[string]bool{} + br := bytes.NewReader(data) + reader := bufio.NewReader(br) + + for { + line, err := reader.ReadString('\n') + ips := strings.Split(line, ",") + + for ipk := range ips { + // Add /32 prefix if no exist, then add to the map + ip := strings.Trim(ips[ipk], "\n, ") + if !strings.Contains(ip, "/") { + ip += "/32" + } + ipmap[ip] = true + } + + if err == io.EOF { + break + } + } + + iplist := make([]net.IPNet, 0, len(ipmap)) + for key, _ := range ipmap { + if _, ipnet, err := net.ParseCIDR(key); err == nil { + iplist = append(iplist, *ipnet) + } + } + + return iplist, nil +} diff --git a/sources/file.go b/sources/file.go new file mode 100644 index 0000000..19c1d0c --- /dev/null +++ b/sources/file.go @@ -0,0 +1,54 @@ +package sources + +import ( + "bufio" + "iplistget/common" + "net" + "os" + "strings" +) + +func init() { + common.RegisterSource(File{FileName: "iplistget.txt"}) +} + +type File struct { + FileName string +} + +func (s File) Get(filename string) ([]net.IPNet, error) { + if filename != "" { + s.FileName = filename + } + + file, err := os.Open(s.FileName) + if err != nil { + return nil, err + } + defer file.Close() + + ipmap := map[string]bool{} + + // TODO Provide custom split function for scanner to be universal + scanner := bufio.NewScanner(file) + for scanner.Scan() { + ip := scanner.Text() + if !strings.Contains(ip, "/") { + ip += "/32" + } + ipmap[ip] = true + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + iplist := make([]net.IPNet, 0, len(ipmap)) + for key, _ := range ipmap { + if _, ipnet, err := net.ParseCIDR(key); err == nil { + iplist = append(iplist, *ipnet) + } + } + + return iplist, nil +} diff --git a/sources/ipdeny.go b/sources/ipdeny.go new file mode 100644 index 0000000..8e7cf3a --- /dev/null +++ b/sources/ipdeny.go @@ -0,0 +1,55 @@ +package sources + +import ( + "bufio" + "errors" + "iplistget/common" + "net" + "strings" +) + +func init() { + common.RegisterSource(IPDeny{}) +} + +type IPDeny struct { + URL string +} + +func (s IPDeny) Get(url string) ([]net.IPNet, error) { + if url != "" { + s.URL = url + } else { + return nil, errors.New("You have to provide a full URL to zone file for this source module, as it don't have a default one!") + } + + reader, err := common.URLReader(s.URL) + if err != nil { + return nil, err + } + defer reader.Close() + + ipmap := map[string]bool{} + + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + ip := scanner.Text() + if !strings.Contains(ip, "/") { + ip += "/32" + } + ipmap[ip] = true + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + iplist := make([]net.IPNet, 0, len(ipmap)) + for key, _ := range ipmap { + if _, ipnet, err := net.ParseCIDR(key); err == nil { + iplist = append(iplist, *ipnet) + } + } + + return iplist, nil +} diff --git a/sources/openbl.go b/sources/openbl.go new file mode 100644 index 0000000..a4043cd --- /dev/null +++ b/sources/openbl.go @@ -0,0 +1,52 @@ +package sources + +import ( + "bufio" + "iplistget/common" + "net" + "strings" +) + +func init() { + common.RegisterSource(OpenBL{URL: "http://www.openbl.org/lists/base_7days.txt"}) +} + +type OpenBL struct { + URL string +} + +func (s OpenBL) Get(url string) ([]net.IPNet, error) { + if url != "" { + s.URL = url + } + + reader, err := common.URLReader(s.URL) + if err != nil { + return nil, err + } + defer reader.Close() + + ipmap := map[string]bool{} + + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + ip := scanner.Text() + if !strings.Contains(ip, "/") { + ip += "/32" + } + ipmap[ip] = true + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + iplist := make([]net.IPNet, 0, len(ipmap)) + for key, _ := range ipmap { + if _, ipnet, err := net.ParseCIDR(key); err == nil { + iplist = append(iplist, *ipnet) + } + } + + return iplist, nil +} diff --git a/sources/test.go b/sources/test.go new file mode 100644 index 0000000..ad19c24 --- /dev/null +++ b/sources/test.go @@ -0,0 +1,24 @@ +package sources + +import ( + "fmt" + "iplistget/common" + "net" +) + +func init() { + common.RegisterSource(Test{}) +} + +type Test struct { +} + +func (s Test) Get(url string) ([]net.IPNet, error) { + iplist := make([]net.IPNet, 0, 254) + for i := 1; i < 255; i++ { + _, ipnet, _ := net.ParseCIDR(fmt.Sprintf("127.255.255.%v/32", i)) + iplist = append(iplist, *ipnet) + } + + return iplist, nil +} diff --git a/targets/file.go b/targets/file.go new file mode 100644 index 0000000..8f44dae --- /dev/null +++ b/targets/file.go @@ -0,0 +1,36 @@ +package targets + +import ( + "fmt" + "iplistget/common" + "net" + "os" +) + +func init() { + common.RegisterTarget(File{FileName: "iplistget.txt"}) +} + +type File struct { + FileName string +} + +func (s File) Set(filename string, iplist []net.IPNet) error { + if filename != "" { + s.FileName = filename + } + + file, err := os.Create(s.FileName) + if err != nil { + return err + } + defer file.Close() + + for ind := range iplist { + if _, err := file.WriteString(fmt.Sprintln(iplist[ind].String())); err != nil { + return err + } + } + + return nil +} diff --git a/targets/ipset.go b/targets/ipset.go new file mode 100644 index 0000000..e0e5fa6 --- /dev/null +++ b/targets/ipset.go @@ -0,0 +1,46 @@ +package targets + +import ( + "fmt" + "io/ioutil" + "iplistget/common" + "net" + "os" + "os/exec" +) + +func init() { + common.RegisterTarget(Ipset{Table: "iplistget"}) +} + +type Ipset struct { + Table string +} + +func (s Ipset) Set(table string, iplist []net.IPNet) error { + if table != "" { + s.Table = table + } + + tmpFile, err := ioutil.TempFile("", "iplistget-") + if err != nil { + return err + } + + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + for key := range iplist { + _, err := tmpFile.WriteString(fmt.Sprintf("add -exist %s %s\n", s.Table, iplist[key].String())) + if err != nil { + return err + } + } + + ipset := exec.Command("ipset", "restore", "-f", tmpFile.Name()) + if err := ipset.Run(); err != nil { + return err + } + + return nil +} diff --git a/targets/stdout.go b/targets/stdout.go new file mode 100644 index 0000000..f7d6c11 --- /dev/null +++ b/targets/stdout.go @@ -0,0 +1,24 @@ +package targets + +import ( + "fmt" + "iplistget/common" + "net" +) + +func init() { + common.RegisterTarget(Stdout{}) +} + +type Stdout struct { +} + +func (s Stdout) Set(destination string, iplist []net.IPNet) error { + for ind := range iplist { + if _, err := fmt.Println(iplist[ind].String()); err != nil { + return err + } + } + + return nil +}