Initial project upload

.gitignore

@@ -0,0 +1,24 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof

README.md

@@ -0,0 +1,21 @@
+# IPListget
+
+IPListget is a network list downloader. It can download a network list from various sources and apply it at different targets. For example, it can download a list from openbl.org and put it in ipset list.
+
+Below you will find the list of available sources and targets.
+
+Sources:
+  - antizapret - antizapret.info list of resources, blocked by Russian government
+  - openbl - openbl.org blacklist of internet abusers
+  - ipdeny - ipdeny.com country block lists
+  - file - reads IPs from a file (one per line)
+  - test - test source generating a list of 127.254.254.0/24 for test purposes
+
+Targets:
+  - ipset - inserts networks to ipset list
+  - file - writes networks to a file (one per line)
+  - stdout - prints a list to stdout
+
+More sources and targets will be added in the future. The following goals are:
+  - more sources of potentially dangerous and unwanted addresses, like tor proxys
+  - targets for different network devices, like routers and firewalls, especially Cisco, Juniper and Mikrotik

common/sources.go

@@ -0,0 +1,23 @@
+package common
+
+import (
+	"net"
+	"reflect"
+	"strings"
+)
+
+var (
+	Sources map[string]SourceModuleType //Available source modules storage
+)
+
+type SourceModuleType interface {
+	Get(url string) ([]net.IPNet, error)
+}
+
+func init() {
+	Sources = make(map[string]SourceModuleType)
+}
+
+func RegisterSource(src SourceModuleType) {
+	Sources[strings.ToLower(reflect.ValueOf(src).Type().Name())] = src
+}

common/targets.go

@@ -0,0 +1,23 @@
+package common
+
+import (
+	"net"
+	"reflect"
+	"strings"
+)
+
+var (
+	Targets map[string]TargetModuleType //Available target modules storage
+)
+
+type TargetModuleType interface {
+	Set(destination string, iplist []net.IPNet) error
+}
+
+func init() {
+	Targets = make(map[string]TargetModuleType)
+}
+
+func RegisterTarget(tgt TargetModuleType) {
+	Targets[strings.ToLower(reflect.ValueOf(tgt).Type().Name())] = tgt
+}

common/utils.go

@@ -0,0 +1,52 @@
+package common
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+)
+
+func FetchURL(url string) ([]byte, error) {
+	resp, err := http.Get(url)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, errors.New(fmt.Sprintf("Request failed, http status: %v", resp.StatusCode))
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
+func URLReader(url string) (io.ReadCloser, error) {
+	resp, err := http.Get(url)
+
+	if resp == nil {
+		return nil, errors.New("Server returned empty response")
+	}
+
+	if err != nil {
+		// Even on error response may be non-nil
+		resp.Body.Close()
+		return nil, err
+	}
+
+	if resp.StatusCode != 200 {
+		resp.Body.Close()
+		return nil, errors.New(fmt.Sprintf("Request failed, http status: %v", resp.StatusCode))
+	}
+
+	return resp.Body, nil
+}

main.go

@@ -0,0 +1,98 @@
+// iplistget project main.go
+// Production build: go build -ldflags="-w" -o iplistget_release.exe iplistget
+package main
+
+import (
+	"flag"
+	"fmt"
+	"iplistget/common"
+	_ "iplistget/sources"
+	_ "iplistget/targets"
+	"net"
+	"os"
+)
+
+const (
+	APP_NAME = "iplistget network lists operator"
+	VERSION  = "0.1-dev"
+)
+
+var (
+	// Application flags
+	showVersion bool // Show version and exit
+	//sudo        bool   // Use sudo for privileged operations
+	listmodules bool   // List available sources
+	source      string // Module name to fetch the list
+	target      string // Module name to supply this list to (iproute, ipset, file, stdout)
+	destination string // Optional parameter where to apply (routing table, ipset list, filename)
+	sourceurl   string // Override the default source url
+)
+
+func init() {
+	flag.BoolVar(&showVersion, "version", false, "Show version and exit")
+	flag.BoolVar(&listmodules, "modules", false, "List all available modules and exit")
+	flag.StringVar(&source, "source", "", "Use specified source module")
+	flag.StringVar(&sourceurl, "url", "", "Override the default URL for source module")
+	flag.StringVar(&target, "target", "", "Use specified target module")
+	flag.StringVar(&destination, "destination", "", "Override the default destination (table name, list name, filename)")
+	//flag.BoolVar(&sudo, "sudo", false, "Use sudo for privileged operations")
+}
+
+func main() {
+	flag.Parse()
+
+	if showVersion {
+		fmt.Printf("%s version %s\n", APP_NAME, VERSION)
+		return
+	}
+
+	if listmodules {
+		lsmodules()
+		return
+	}
+
+	if source == "" {
+		fmt.Println("No source module specified")
+		os.Exit(1)
+	}
+
+	if target == "" {
+		fmt.Println("No destination module provided")
+		os.Exit(1)
+	}
+
+	srcmod, srcok := common.Sources[source]
+	if srcok != true {
+		fmt.Printf("Error. No such source module `%v`!\n", source)
+		os.Exit(1)
+	}
+
+	tgtmod, tgtok := common.Targets[target]
+	if tgtok != true {
+		fmt.Printf("Error. No such target module `%v`!\n", target)
+		os.Exit(1)
+	}
+
+	var iplist []net.IPNet
+	iplist, geterr := srcmod.Get(sourceurl)
+	if geterr != nil {
+		fmt.Printf("Error. Source module failed. %v\n", geterr)
+	}
+
+	if err := tgtmod.Set(destination, iplist); err != nil {
+		fmt.Printf("Error. Target module failed. %v\n", err)
+	}
+}
+
+func lsmodules() {
+	fmt.Print("Available sources: ")
+	for src := range common.Sources {
+		fmt.Print(src, " ")
+	}
+
+	fmt.Print("\nAvailable targets: ")
+	for tgt := range common.Targets {
+		fmt.Print(tgt, " ")
+	}
+	fmt.Println()
+}

sources/antizapret.go

@@ -0,0 +1,60 @@
+package sources
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"iplistget/common"
+	"net"
+	"strings"
+)
+
+func init() {
+	common.RegisterSource(Antizapret{URL: "https://api.antizapret.info/group.php?data=ip"})
+}
+
+type Antizapret struct {
+	URL string
+}
+
+func (s Antizapret) Get(url string) ([]net.IPNet, error) {
+	if url != "" {
+		s.URL = url
+	}
+
+	data, err := common.FetchURL(s.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	ipmap := map[string]bool{}
+	br := bytes.NewReader(data)
+	reader := bufio.NewReader(br)
+
+	for {
+		line, err := reader.ReadString('\n')
+		ips := strings.Split(line, ",")
+
+		for ipk := range ips {
+			// Add /32 prefix if no exist, then add to the map
+			ip := strings.Trim(ips[ipk], "\n, ")
+			if !strings.Contains(ip, "/") {
+				ip += "/32"
+			}
+			ipmap[ip] = true
+		}
+
+		if err == io.EOF {
+			break
+		}
+	}
+
+	iplist := make([]net.IPNet, 0, len(ipmap))
+	for key, _ := range ipmap {
+		if _, ipnet, err := net.ParseCIDR(key); err == nil {
+			iplist = append(iplist, *ipnet)
+		}
+	}
+
+	return iplist, nil
+}

sources/file.go

@@ -0,0 +1,54 @@
+package sources
+
+import (
+	"bufio"
+	"iplistget/common"
+	"net"
+	"os"
+	"strings"
+)
+
+func init() {
+	common.RegisterSource(File{FileName: "iplistget.txt"})
+}
+
+type File struct {
+	FileName string
+}
+
+func (s File) Get(filename string) ([]net.IPNet, error) {
+	if filename != "" {
+		s.FileName = filename
+	}
+
+	file, err := os.Open(s.FileName)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	ipmap := map[string]bool{}
+
+	// TODO Provide custom split function for scanner to be universal
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		ip := scanner.Text()
+		if !strings.Contains(ip, "/") {
+			ip += "/32"
+		}
+		ipmap[ip] = true
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	iplist := make([]net.IPNet, 0, len(ipmap))
+	for key, _ := range ipmap {
+		if _, ipnet, err := net.ParseCIDR(key); err == nil {
+			iplist = append(iplist, *ipnet)
+		}
+	}
+
+	return iplist, nil
+}

sources/ipdeny.go

@@ -0,0 +1,55 @@
+package sources
+
+import (
+	"bufio"
+	"errors"
+	"iplistget/common"
+	"net"
+	"strings"
+)
+
+func init() {
+	common.RegisterSource(IPDeny{})
+}
+
+type IPDeny struct {
+	URL string
+}
+
+func (s IPDeny) Get(url string) ([]net.IPNet, error) {
+	if url != "" {
+		s.URL = url
+	} else {
+		return nil, errors.New("You have to provide a full URL to zone file for this source module, as it don't have a default one!")
+	}
+
+	reader, err := common.URLReader(s.URL)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+
+	ipmap := map[string]bool{}
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		ip := scanner.Text()
+		if !strings.Contains(ip, "/") {
+			ip += "/32"
+		}
+		ipmap[ip] = true
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	iplist := make([]net.IPNet, 0, len(ipmap))
+	for key, _ := range ipmap {
+		if _, ipnet, err := net.ParseCIDR(key); err == nil {
+			iplist = append(iplist, *ipnet)
+		}
+	}
+
+	return iplist, nil
+}

sources/openbl.go

@@ -0,0 +1,52 @@
+package sources
+
+import (
+	"bufio"
+	"iplistget/common"
+	"net"
+	"strings"
+)
+
+func init() {
+	common.RegisterSource(OpenBL{URL: "http://www.openbl.org/lists/base_7days.txt"})
+}
+
+type OpenBL struct {
+	URL string
+}
+
+func (s OpenBL) Get(url string) ([]net.IPNet, error) {
+	if url != "" {
+		s.URL = url
+	}
+
+	reader, err := common.URLReader(s.URL)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+
+	ipmap := map[string]bool{}
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		ip := scanner.Text()
+		if !strings.Contains(ip, "/") {
+			ip += "/32"
+		}
+		ipmap[ip] = true
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	iplist := make([]net.IPNet, 0, len(ipmap))
+	for key, _ := range ipmap {
+		if _, ipnet, err := net.ParseCIDR(key); err == nil {
+			iplist = append(iplist, *ipnet)
+		}
+	}
+
+	return iplist, nil
+}

sources/test.go

@@ -0,0 +1,24 @@
+package sources
+
+import (
+	"fmt"
+	"iplistget/common"
+	"net"
+)
+
+func init() {
+	common.RegisterSource(Test{})
+}
+
+type Test struct {
+}
+
+func (s Test) Get(url string) ([]net.IPNet, error) {
+	iplist := make([]net.IPNet, 0, 254)
+	for i := 1; i < 255; i++ {
+		_, ipnet, _ := net.ParseCIDR(fmt.Sprintf("127.255.255.%v/32", i))
+		iplist = append(iplist, *ipnet)
+	}
+
+	return iplist, nil
+}

targets/file.go

@@ -0,0 +1,36 @@
+package targets
+
+import (
+	"fmt"
+	"iplistget/common"
+	"net"
+	"os"
+)
+
+func init() {
+	common.RegisterTarget(File{FileName: "iplistget.txt"})
+}
+
+type File struct {
+	FileName string
+}
+
+func (s File) Set(filename string, iplist []net.IPNet) error {
+	if filename != "" {
+		s.FileName = filename
+	}
+
+	file, err := os.Create(s.FileName)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	for ind := range iplist {
+		if _, err := file.WriteString(fmt.Sprintln(iplist[ind].String())); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

targets/ipset.go

@@ -0,0 +1,46 @@
+package targets
+
+import (
+	"fmt"
+	"io/ioutil"
+	"iplistget/common"
+	"net"
+	"os"
+	"os/exec"
+)
+
+func init() {
+	common.RegisterTarget(Ipset{Table: "iplistget"})
+}
+
+type Ipset struct {
+	Table string
+}
+
+func (s Ipset) Set(table string, iplist []net.IPNet) error {
+	if table != "" {
+		s.Table = table
+	}
+
+	tmpFile, err := ioutil.TempFile("", "iplistget-")
+	if err != nil {
+		return err
+	}
+
+	defer os.Remove(tmpFile.Name())
+	defer tmpFile.Close()
+
+	for key := range iplist {
+		_, err := tmpFile.WriteString(fmt.Sprintf("add -exist %s %s\n", s.Table, iplist[key].String()))
+		if err != nil {
+			return err
+		}
+	}
+
+	ipset := exec.Command("ipset", "restore", "-f", tmpFile.Name())
+	if err := ipset.Run(); err != nil {
+		return err
+	}
+
+	return nil
+}

targets/stdout.go

@@ -0,0 +1,24 @@
+package targets
+
+import (
+	"fmt"
+	"iplistget/common"
+	"net"
+)
+
+func init() {
+	common.RegisterTarget(Stdout{})
+}
+
+type Stdout struct {
+}
+
+func (s Stdout) Set(destination string, iplist []net.IPNet) error {
+	for ind := range iplist {
+		if _, err := fmt.Println(iplist[ind].String()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}