diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Makefile | 5 | ||||
| -rw-r--r-- | conf/k8sCorefile | 13 | ||||
| -rw-r--r-- | core/directives.go | 1 | ||||
| -rw-r--r-- | core/setup/kubernetes.go | 93 | ||||
| -rw-r--r-- | middleware/kubernetes/README.md | 284 | ||||
| -rw-r--r-- | middleware/kubernetes/handler.go | 101 | ||||
| -rw-r--r-- | middleware/kubernetes/k8sclient/dataobjects.go | 110 | ||||
| -rw-r--r-- | middleware/kubernetes/k8sclient/k8sclient.go | 117 | ||||
| -rw-r--r-- | middleware/kubernetes/kubernetes.go | 223 | ||||
| -rw-r--r-- | middleware/kubernetes/lookup.go | 305 | ||||
| -rw-r--r-- | middleware/kubernetes/msg/service.go | 166 | ||||
| -rw-r--r-- | middleware/kubernetes/msg/service_test.go | 125 | ||||
| -rw-r--r-- | middleware/kubernetes/path.go | 17 |
14 files changed, 1560 insertions, 1 deletions
diff --git a/.gitignore b/.gitignore index fe121cbce..362016d3b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ query.log Corefile *.swp coredns +conf/devk8sCorefile @@ -1,5 +1,8 @@ +#VERBOSE := +VERBOSE := -v + all: - go build + go build $(VERBOSE) .PHONY: docker docker: diff --git a/conf/k8sCorefile b/conf/k8sCorefile new file mode 100644 index 000000000..7825a626e --- /dev/null +++ b/conf/k8sCorefile @@ -0,0 +1,13 @@ +# Serve on port 53 +.:53 { + # use kubernetes middleware for domain "coredns.local" + kubernetes coredns.local { + # Use url for k8s API endpoint + endpoint http://localhost:8080 + } + # Perform DNS response caching for the coredns.local zone + # Cache timeout is provided by the integer in seconds + # This works for the kubernetes middleware.) + #cache 20 coredns.local + #cache 160 coredns.local +} diff --git a/core/directives.go b/core/directives.go index 3de69f8fb..63e245578 100644 --- a/core/directives.go +++ b/core/directives.go @@ -65,6 +65,7 @@ var directiveOrder = []directive{ {"file", setup.File}, {"secondary", setup.Secondary}, {"etcd", setup.Etcd}, + {"kubernetes", setup.Kubernetes}, {"proxy", setup.Proxy}, } diff --git a/core/setup/kubernetes.go b/core/setup/kubernetes.go new file mode 100644 index 000000000..8f21286f5 --- /dev/null +++ b/core/setup/kubernetes.go @@ -0,0 +1,93 @@ +package setup + +import ( +// "crypto/tls" +// "crypto/x509" + "fmt" +// "io/ioutil" +// "net" +// "net/http" +// "time" + + "github.com/miekg/coredns/middleware" + "github.com/miekg/coredns/middleware/kubernetes" + k8sc "github.com/miekg/coredns/middleware/kubernetes/k8sclient" + "github.com/miekg/coredns/middleware/proxy" +// "github.com/miekg/coredns/middleware/singleflight" + + "golang.org/x/net/context" +) + +const defaultK8sEndpoint = "http://localhost:8080" + +// Kubernetes sets up the kubernetes middleware. +func Kubernetes(c *Controller) (middleware.Middleware, error) { + fmt.Println("controller %v", c) + // TODO: Determine if subzone support required + + kubernetes, err := kubernetesParse(c) + + if err != nil { + return nil, err + } + + return func(next middleware.Handler) middleware.Handler { + kubernetes.Next = next + return kubernetes + }, nil +} + +func kubernetesParse(c *Controller) (kubernetes.Kubernetes, error) { + + /* + * TODO: Remove unused state and simplify. + * Inflight and Ctx might not be needed. Leaving in place until + * we take a pass at API caching and optimizing connector to the + * k8s API. Single flight (or limited upper-bound) for inflight + * API calls may be desirable. + */ + + k8s := kubernetes.Kubernetes{ + Proxy: proxy.New([]string{}), + Ctx: context.Background(), +// Inflight: &singleflight.Group{}, + APIConn: nil, + } + var ( + endpoints = []string{defaultK8sEndpoint} + ) + for c.Next() { + if c.Val() == "kubernetes" { + k8s.Zones = c.RemainingArgs() + if len(k8s.Zones) == 0 { + k8s.Zones = c.ServerBlockHosts + } + middleware.Zones(k8s.Zones).FullyQualify() + if c.NextBlock() { + // TODO(miek): 2 switches? + switch c.Val() { + case "endpoint": + args := c.RemainingArgs() + if len(args) == 0 { + return kubernetes.Kubernetes{}, c.ArgErr() + } + endpoints = args + k8s.APIConn = k8sc.NewK8sConnector(endpoints[0]) + } + for c.Next() { + switch c.Val() { + case "endpoint": + args := c.RemainingArgs() + if len(args) == 0 { + return kubernetes.Kubernetes{}, c.ArgErr() + } + endpoints = args + } + } + } + return k8s, nil + } + fmt.Println("endpoints='%v'", endpoints) + } + return kubernetes.Kubernetes{}, nil +} diff --git a/middleware/kubernetes/README.md b/middleware/kubernetes/README.md new file mode 100644 index 000000000..7e2e42dc2 --- /dev/null +++ b/middleware/kubernetes/README.md @@ -0,0 +1,284 @@ +# kubernetes + +`kubernetes` enables reading zone data from a kubernetes cluster. Record names +are constructed as "myservice.mynamespace.coredns.local" where: + +* "myservice" is the name of the k8s service (this may include multiple DNS labels, such as "c1.myservice"), +* "mynamespace" is the k8s namespace for the service, and +* "coredns.local" is the zone configured for `kubernetes`. + + +## Syntax + +~~~ +kubernetes [zones...] +~~~ + +* `zones` zones kubernetes should be authorative for. + + +~~~ +kubernetes [zones] { + endpoint http://localhost:8080 +} +~~~ + +* `endpoint` the kubernetes API endpoint, default to http://localhost:8080 + +## Examples + +This is the default kubernetes setup, with everything specified in full: + +~~~ +# Serve on port 53 +.:53 { + # use kubernetes middleware for domain "coredns.local" + kubernetes coredns.local { + # Use url for k8s API endpoint + endpoint http://localhost:8080 + } +# cache 160 coredns.local +} +~~~ + +### Basic Setup + +#### Launch Kubernetes + +Kubernetes is launched using the commands in the following `run_k8s.sh` script: + +~~~ +#!/bin/bash + +# Based on instructions at: http://kubernetes.io/docs/getting-started-guides/docker/ + +#K8S_VERSION=$(curl -sS https://storage.googleapis.com/kubernetes-release/release/latest.txt) +K8S_VERSION="v1.2.4" + +ARCH="amd64" + +export K8S_VERSION +export ARCH + +#DNS_ARGUMENTS="--cluster-dns=10.0.0.10 --cluster-domain=cluster.local" +DNS_ARGUMENTS="" + +docker run -d \ + --volume=/:/rootfs:ro \ + --volume=/sys:/sys:ro \ + --volume=/var/lib/docker/:/var/lib/docker:rw \ + --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ + --volume=/var/run:/var/run:rw \ + --net=host \ + --pid=host \ + --privileged \ + gcr.io/google_containers/hyperkube-${ARCH}:${K8S_VERSION} \ + /hyperkube kubelet \ + --containerized \ + --hostname-override=127.0.0.1 \ + --api-servers=http://localhost:8080 \ + --config=/etc/kubernetes/manifests \ + ${DNS_ARGUMENTS} \ + --allow-privileged --v=2 +~~~ + +#### Configure kubectl and test + +The kubernetes control client can be downloaded from the generic URL: +`http://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${GOOS}/${GOARCH}/${K8S_BINARY}` + +For example, the kubectl client for Linux can be downloaded using the command: +`curl -sSL "http://storage.googleapis.com/kubernetes-release/release/v1.2.4/bin/linux/amd64/kubectl" + +The following `setup_kubectl.sh` script can be stored in the same directory as +kubectl to setup +kubectl to communicate with kubernetes running on the localhost: + +~~~ +#!/bin/bash + +BASEDIR=`realpath $(dirname ${0})` + +${BASEDIR}/kubectl config set-cluster test-doc --server=http://localhost:8080 +${BASEDIR}/kubectl config set-context test-doc --cluster=test-doc +${BASEDIR}/kubectl config use-context test-doc + +alias kubctl="${BASEDIR}/kubectl" +~~~ + + +Verify that kubectl is working by querying for the kubernetes namespaces: + +~~~ +$ ./kubectl get namespaces +NAME STATUS AGE +default Active 8d +test Active 7d +~~~ + + +#### Launch a kubernetes service and expose the service + +The following commands will create a kubernetes namespace "demo", +launch an nginx service in the namespace, and expose the service on port 80: + +~~~ +$ ./kubectl create namespace demo +$ ./kubectl get namespace + +$ ./kubectl run mynginx --namespace=demo --image=nginx +$ /kubectl get deployment --namespace=demo + +$ ./kubectl expose deployment mynginx --namespace=demo --port=80 +$ ./kubectl get service --namespace=demo +~~~ + + +#### Launch CoreDNS + +Build CoreDNS and launch using the configuration file in `conf/k8sCorefile`. +This configuration file sets up CoreDNS to use the zone `coredns.local` for +the kubernetes services. + +The command to launch CoreDNS is: + +~~~ +$ ./coredns -conf conf/k8sCoreFile +~~~ + +In a separate terminal a dns query can be issued using dig: + +~~~ +$ dig @localhost mynginx.demo.coredns.local + +; <<>> DiG 9.9.4-RedHat-9.9.4-29.el7_2.3 <<>> @localhost mynginx.demo.coredns.local +; (2 servers found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 47614 +;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 + +;; OPT PSEUDOSECTION: +; EDNS: version: 0, flags:; udp: 4096 +;; QUESTION SECTION: +;mynginx.demo.coredns.local. IN A + +;; ANSWER SECTION: +mynginx.demo.coredns.local. 0 IN A 10.0.0.10 + +;; Query time: 2 msec +;; SERVER: ::1#53(::1) +;; WHEN: Thu Jun 02 11:07:18 PDT 2016 +;; MSG SIZE rcvd: 71 +~~~ + + + +## Implementation Notes/Ideas + +### Basic Zone Mapping (implemented) +The middleware is configured with a "zone" string. For +example: "zone = coredns.local". + +The Kubernetes service "myservice" running in "mynamespace" would map +to: "myservice.mynamespace.coredns.local". + +The middleware should publish an A record for that service and a service record. + +Initial implementation just performs the above simple mapping. Subsequent +revisions should allow different namespaces to be published under different zones. + +For example: + + # Serve on port 53 + .:53 { + # use kubernetes middleware for domain "coredns.local" + kubernetes coredns.local { + # Use url for k8s API endpoint + endpoint http://localhost:8080 + } + # Perform DNS response caching for the coredns.local zone + # Cache timeout is provided by the integer argument in seconds + # This works for the kubernetes middleware.) + #cache 20 coredns.local + #cache 160 coredns.local + } + + +### Internal IP or External IP? +* Should the Corefile configuration allow control over whether the internal IP or external IP is exposed? +* If the Corefile configuration allows control over internal IP or external IP, then the config should allow users to control the precidence. + +For example a service "myservice" running in namespace "mynamespace" with internal IP "10.0.0.100" and external IP "1.2.3.4". + +This example could be published as: + +| Corefile directive | Result | +|------------------------------|---------------------| +| iporder = internal | 10.0.0.100 | +| iporder = external | 1.2.3.4 | +| iporder = external, internal | 10.0.0.100, 1.2.3.4 | +| iporder = internal, external | 1.2.3.4, 10.0.0.100 | +| _no directive_ | 10.0.0.100, 1.2.3.4 | + + +### Wildcards + +Publishing DNS records for singleton services isn't very interesting. Service +names are unique within a k8s namespace therefore multiple services will be +commonly run with a structured naming scheme. + +For example, running multiple nginx services under the names: + +| Service name | +|--------------| +| c1.nginx | +| c2.nginx | + +or: + +| Service name | +|--------------| +| nginx.c3 | +| nginx.c4 | + +A DNS query with wildcard support for "nginx" in these examples should +return the IP addresses for all services with "nginx" in the service name. + +TBD: +* How does this relate the the k8s load-balancer configuration? +* Do wildcards search across namespaces? +* Initial implementation assumes that a namespace maps to the first DNS label below the zone managed by the kubernetes middleware. This assumption may need to be revised. + + +## TODO +* Implement namespace filtering to different zones. +* Implement IP selection and ordering (internal/external). +* Implement SRV-record queries using naive lookup. +* Flatten service and namespace names to valid DNS characters. (service names + and namespace names in k8s may use uppercase and non-DNS characters. Implement + flattening to lower case and mapping of non-DNS characters to DNS characters + in a standard way.) +* Do we need to generate synthetic zone records for namespaces? +* Implement wildcard-based lookup. +* Improve lookup to reduce size of query result obtained from k8s API. + (namespace-based?, other ideas?) +* How to support label specification in Corefile to allow use of labels to + indicate zone? (Is this even useful?) For example, the following configuration + exposes all services labeled for the "staging" environment and tenant "customerB" + in the zone "customerB.stage.local": + +~~~ +kubernetes customerB.stage.local { + # Use url for k8s API endpoint + endpoint http://localhost:8080 + label "environment" : "staging", "tenant" : "customerB" +} +~~~ + +* Test with CoreDNS caching. CoreDNS caching for DNS response is working using + the `cache` directive. Tested working using 20s cache timeout and A-record queries. +* DNS response caching is good, but we should also cache at the http query + level as well. (Take a look at https://github.com/patrickmn/go-cache as + a potential expiring cache implementation for the http API queries.) + diff --git a/middleware/kubernetes/handler.go b/middleware/kubernetes/handler.go new file mode 100644 index 000000000..44de8da3d --- /dev/null +++ b/middleware/kubernetes/handler.go @@ -0,0 +1,101 @@ +package kubernetes + +import ( + "fmt" + + "github.com/miekg/coredns/middleware" + + "github.com/miekg/dns" + "golang.org/x/net/context" +) + +func (k Kubernetes) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) { + + fmt.Println("[debug] here entering ServeDNS: ctx:%v dnsmsg:%v", ctx, r) + + state := middleware.State{W: w, Req: r} + if state.QClass() != dns.ClassINET { + return dns.RcodeServerFailure, fmt.Errorf("can only deal with ClassINET") + } + + // Check that query matches one of the zones served by this middleware, + // otherwise delegate to the next in the pipeline. + zone := middleware.Zones(k.Zones).Matches(state.Name()) + if zone == "" { + if k.Next == nil { + return dns.RcodeServerFailure, nil + } + return k.Next.ServeDNS(ctx, w, r) + } + + m := new(dns.Msg) + m.SetReply(r) + m.Authoritative, m.RecursionAvailable, m.Compress = true, true, true + + var ( + records, extra []dns.RR + err error + ) + switch state.Type() { + case "A": + records, err = k.A(zone, state, nil) + case "AAAA": + records, err = k.AAAA(zone, state, nil) + case "TXT": + records, err = k.TXT(zone, state) + case "CNAME": + records, err = k.CNAME(zone, state) + case "MX": + records, extra, err = k.MX(zone, state) + case "SRV": + records, extra, err = k.SRV(zone, state) + case "SOA": + records = []dns.RR{k.SOA(zone, state)} + case "NS": + if state.Name() == zone { + records, extra, err = k.NS(zone, state) + break + } + fallthrough + default: + // Do a fake A lookup, so we can distinguish betwen NODATA and NXDOMAIN + _, err = k.A(zone, state, nil) + } + if isKubernetesNameError(err) { + return k.Err(zone, dns.RcodeNameError, state) + } + if err != nil { + return dns.RcodeServerFailure, err + } + + if len(records) == 0 { + return k.Err(zone, dns.RcodeSuccess, state) + } + + m.Answer = append(m.Answer, records...) + m.Extra = append(m.Extra, extra...) + + m = dedup(m) + state.SizeAndDo(m) + m, _ = state.Scrub(m) + w.WriteMsg(m) + return dns.RcodeSuccess, nil +} + +// NoData write a nodata response to the client. +func (k Kubernetes) Err(zone string, rcode int, state middleware.State) (int, error) { + m := new(dns.Msg) + m.SetRcode(state.Req, rcode) + m.Ns = []dns.RR{k.SOA(zone, state)} + state.SizeAndDo(m) + state.W.WriteMsg(m) + return rcode, nil +} + +func dedup(m *dns.Msg) *dns.Msg { + // TODO(miek): expensive! + m.Answer = dns.Dedup(m.Answer, nil) + m.Ns = dns.Dedup(m.Ns, nil) + m.Extra = dns.Dedup(m.Extra, nil) + return m +} diff --git a/middleware/kubernetes/k8sclient/dataobjects.go b/middleware/kubernetes/k8sclient/dataobjects.go new file mode 100644 index 000000000..a5ab4f19c --- /dev/null +++ b/middleware/kubernetes/k8sclient/dataobjects.go @@ -0,0 +1,110 @@ +package k8sclient + +import ( + "encoding/json" + "net/http" +) + + +func getJson(url string, target interface{}) error { + r, err := http.Get(url) + if err != nil { + return err + } + defer r.Body.Close() + + return json.NewDecoder(r.Body).Decode(target) +} + + +// Kubernetes Resource List +type ResourceList struct { + Kind string `json:"kind"` + GroupVersion string `json:"groupVersion"` + Resources []resource `json:"resources"` +} + +type resource struct { + Name string `json:"name"` + Namespaced bool `json:"namespaced"` + Kind string `json:"kind"` +} + + +// Kubernetes NamespaceList +type NamespaceList struct { + Kind string `json:"kind"` + APIVersion string `json:"apiVersion"` + Metadata apiListMetadata `json:"metadata"` + Items []nsItems `json:"items"` +} + +type apiListMetadata struct { + SelfLink string `json:"selfLink"` + resourceVersion string `json:"resourceVersion"` +} + +type nsItems struct { + Metadata nsMetadata `json:"metadata"` + Spec nsSpec `json:"spec"` + Status nsStatus `json:"status"` +} + +type nsMetadata struct { + Name string `json:"name"` + SelfLink string `json:"selfLink"` + Uid string `json:"uid"` + ResourceVersion string `json:"resourceVersion"` + CreationTimestamp string `json:"creationTimestamp"` +} + +type nsSpec struct { + Finalizers []string `json:"finalizers"` +} + +type nsStatus struct { + Phase string `json:"phase"` +} + + +// Kubernetes ServiceList +type ServiceList struct { + Kind string `json:"kind"` + APIVersion string `json:"apiVersion"` + Metadata apiListMetadata `json:"metadata"` + Items []ServiceItem `json:"items"` +} + +type ServiceItem struct { + Metadata serviceMetadata `json:"metadata"` + Spec serviceSpec `json:"spec"` +// Status serviceStatus `json:"status"` +} + +type serviceMetadata struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + SelfLink string `json:"selfLink"` + Uid string `json:"uid"` + ResourceVersion string `json:"resourceVersion"` + CreationTimestamp string `json:"creationTimestamp"` + // labels +} + +type serviceSpec struct { + Ports []servicePort `json:"ports"` + ClusterIP string `json:"clusterIP"` + Type string `json:"type"` + SessionAffinity string `json:"sessionAffinity"` +} + +type servicePort struct { + Name string `json:"name"` + Protocol string `json:"protocol"` + Port int `json:"port"` + TargetPort int `json:"targetPort"` +} + +type serviceStatus struct { + LoadBalancer string `json:"loadBalancer"` +} diff --git a/middleware/kubernetes/k8sclient/k8sclient.go b/middleware/kubernetes/k8sclient/k8sclient.go new file mode 100644 index 000000000..a05ef8905 --- /dev/null +++ b/middleware/kubernetes/k8sclient/k8sclient.go @@ -0,0 +1,117 @@ +package k8sclient + +import ( +// "fmt" + "net/url" +) + +// API strings +const ( + apiBase = "/api/v1" + apiNamespaces = "/namespaces" + apiServices = "/services" +) + +// Defaults +const ( + defaultBaseUrl = "http://localhost:8080" +) + + +type K8sConnector struct { + baseUrl string +} + +func (c *K8sConnector) SetBaseUrl(u string) error { + validUrl, error := url.Parse(u) + + if error != nil { + return error + } + c.baseUrl = validUrl.String() + + return nil +} + +func (c *K8sConnector) GetBaseUrl() string { + return c.baseUrl +} + + +func (c *K8sConnector) GetResourceList() *ResourceList { + resources := new(ResourceList) + + error := getJson((c.baseUrl + apiBase), resources) + if error != nil { + return nil + } + + return resources +} + + +func (c *K8sConnector) GetNamespaceList() *NamespaceList { + namespaces := new(NamespaceList) + + error := getJson((c.baseUrl + apiBase + apiNamespaces), namespaces) + if error != nil { + return nil + } + + return namespaces +} + + +func (c *K8sConnector) GetServiceList() *ServiceList { + services := new(ServiceList) + + error := getJson((c.baseUrl + apiBase + apiServices), services) + if error != nil { + return nil + } + + return services +} + + +func (c *K8sConnector) GetServicesByNamespace() map[string][]ServiceItem { + // GetServicesByNamespace returns a map of namespacename :: [ kubernetesServiceItem ] + + items := make(map[string][]ServiceItem) + + k8sServiceList := c.GetServiceList() + k8sItemList := k8sServiceList.Items + + for _, i := range k8sItemList { + namespace := i.Metadata.Namespace + items[namespace] = append(items[namespace], i) + } + + return items +} + + +func (c *K8sConnector) GetServiceItemInNamespace(namespace string, servicename string) *ServiceItem { + // GetServiceItemInNamespace returns the ServiceItem that matches servicename in the namespace + + itemMap := c.GetServicesByNamespace() + + // TODO: Handle case where namesapce == nil + + for _, x := range itemMap[namespace] { + if x.Metadata.Name == servicename { + return &x + } + } + + // No matching item found in namespace + return nil +} + + +func NewK8sConnector(baseurl string) *K8sConnector { + k := new(K8sConnector) + k.SetBaseUrl(baseurl) + + return k +} diff --git a/middleware/kubernetes/kubernetes.go b/middleware/kubernetes/kubernetes.go new file mode 100644 index 000000000..25c8cab3c --- /dev/null +++ b/middleware/kubernetes/kubernetes.go @@ -0,0 +1,223 @@ +// Package kubernetes provides the kubernetes backend. +package kubernetes + +import ( + "fmt" + "strings" + "time" + + "github.com/miekg/coredns/middleware" + "github.com/miekg/coredns/middleware/kubernetes/msg" + k8sc "github.com/miekg/coredns/middleware/kubernetes/k8sclient" + "github.com/miekg/coredns/middleware/proxy" +// "github.com/miekg/coredns/middleware/singleflight" + + "github.com/miekg/dns" + "golang.org/x/net/context" +) + +type Kubernetes struct { + Next middleware.Handler + Zones []string + Proxy proxy.Proxy // Proxy for looking up names during the resolution process + Ctx context.Context +// Inflight *singleflight.Group + APIConn *k8sc.K8sConnector +} + + +func (g Kubernetes) getZoneForName(name string) (string, []string) { + /* + * getZoneForName returns the zone string that matches the name and a + * list of the DNS labels from name that are within the zone. + * For example, if "coredns.local" is a zone configured for the + * Kubernetes middleware, then getZoneForName("a.b.coredns.local") + * will return ("coredns.local", ["a", "b"]). + */ + var zone string + var serviceSegments []string + + for _, z := range g.Zones { + if dns.IsSubDomain(z, name) { + zone = z + + serviceSegments = dns.SplitDomainName(name) + serviceSegments = serviceSegments[:len(serviceSegments) - dns.CountLabel(zone)] + break + } + } + + return zone, serviceSegments +} + + +// Records looks up services in kubernetes. +// If exact is true, it will lookup just +// this name. This is used when find matches when completing SRV lookups +// for instance. +func (g Kubernetes) Records(name string, exact bool) ([]msg.Service, error) { + + fmt.Println("enter Records('", name, "', ", exact, ")") + + zone, serviceSegments := g.getZoneForName(name) + + var serviceName string + var namespace string + + // For initial implementation, assume namespace is first serviceSegment + // and service name is remaining segments. + serviceSegLen := len(serviceSegments) + if serviceSegLen >= 2 { + namespace = serviceSegments[serviceSegLen-1] + serviceName = strings.Join(serviceSegments[:serviceSegLen-1], ".") + } + // else we are looking up the zone. So handle the NS, SOA records etc. + + fmt.Println("[debug] zone: ", zone) + fmt.Println("[debug] servicename: ", serviceName) + fmt.Println("[debug] namespace: ", namespace) + fmt.Println("[debug] APIconn: ", g.APIConn) + + k8sItem := g.APIConn.GetServiceItemInNamespace(namespace, serviceName) + fmt.Println("[debug] k8s item:", k8sItem) + + switch { + case exact && k8sItem == nil: + fmt.Println("here2") + return nil, nil + } + + if k8sItem == nil { + // Did not find item in k8s + return nil, nil + } + + fmt.Println("[debug] clusterIP:", k8sItem.Spec.ClusterIP) + + for _, p := range k8sItem.Spec.Ports { + fmt.Println("[debug] host:", name) + fmt.Println("[debug] port:", p.Port) + } + + clusterIP := k8sItem.Spec.ClusterIP + var records []msg.Service + for _, p := range k8sItem.Spec.Ports{ + s := msg.Service{Host: clusterIP, Port: p.Port} + records = append(records, s) + } + + return records, nil +} + +/* +// Get performs the call to the Kubernetes http API. +func (g Kubernetes) Get(path string, recursive bool) (bool, error) { + + fmt.Println("[debug] in Get path: ", path) + fmt.Println("[debug] in Get recursive: ", recursive) + + return false, nil +} +*/ + +func (g Kubernetes) splitDNSName(name string) []string { + l := dns.SplitDomainName(name) + + for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 { + l[i], l[j] = l[j], l[i] + } + + return l +} + +// skydns/local/skydns/east/staging/web +// skydns/local/skydns/west/production/web +// +// skydns/local/skydns/*/*/web +// skydns/local/skydns/*/web + +// loopNodes recursively loops through the nodes and returns all the values. The nodes' keyname +// will be match against any wildcards when star is true. +/* +func (g Kubernetes) loopNodes(ns []*etcdc.Node, nameParts []string, star bool, bx map[msg.Service]bool) (sx []msg.Service, err error) { + if bx == nil { + bx = make(map[msg.Service]bool) + } +Nodes: + for _, n := range ns { + if n.Dir { + nodes, err := g.loopNodes(n.Nodes, nameParts, star, bx) + if err != nil { + return nil, err + } + sx = append(sx, nodes...) + continue + } + if star { + keyParts := strings.Split(n.Key, "/") + for i, n := range nameParts { + if i > len(keyParts)-1 { + // name is longer than key + continue Nodes + } + if n == "*" || n == "any" { + continue + } + if keyParts[i] != n { + continue Nodes + } + } + } + serv := new(msg.Service) + if err := json.Unmarshal([]byte(n.Value), serv); err != nil { + return nil, err + } + b := msg.Service{Host: serv.Host, Port: serv.Port, Priority: serv.Priority, Weight: serv.Weight, Text: serv.Text, Key: n.Key} + if _, ok := bx[b]; ok { + continue + } + bx[b] = true + + serv.Key = n.Key + serv.Ttl = g.Ttl(n, serv) + if serv.Priority == 0 { + serv.Priority = priority + } + sx = append(sx, *serv) + } + return sx, nil +} + +// Ttl returns the smaller of the kubernetes TTL and the service's +// TTL. If neither of these are set (have a zero value), a default is used. +func (g Kubernetes) Ttl(node *etcdc.Node, serv *msg.Service) uint32 { + kubernetesTtl := uint32(node.TTL) + + if kubernetesTtl == 0 && serv.Ttl == 0 { + return ttl + } + if kubernetesTtl == 0 { + return serv.Ttl + } + if serv.Ttl == 0 { + return kubernetesTtl + } + if kubernetesTtl < serv.Ttl { + return kubernetesTtl + } + return serv.Ttl +} +*/ + +// kubernetesNameError checks if the error is ErrorCodeKeyNotFound from kubernetes. +func isKubernetesNameError(err error) bool { + return false +} + +const ( + priority = 10 // default priority when nothing is set + ttl = 300 // default ttl when nothing is set + minTtl = 60 + hostmaster = "hostmaster" + k8sTimeout = 5 * time.Second +) diff --git a/middleware/kubernetes/lookup.go b/middleware/kubernetes/lookup.go new file mode 100644 index 000000000..1efec7475 --- /dev/null +++ b/middleware/kubernetes/lookup.go @@ -0,0 +1,305 @@ +package kubernetes + +import ( + "fmt" + "math" + "net" + "time" + + "github.com/miekg/coredns/middleware" + "github.com/miekg/coredns/middleware/kubernetes/msg" + + "github.com/miekg/dns" +) + +func (k Kubernetes) records(state middleware.State, exact bool) ([]msg.Service, error) { + services, err := k.Records(state.Name(), exact) + if err != nil { + return nil, err + } + // TODO: Do we want to support the SkyDNS (hacky) Group feature? + services = msg.Group(services) + return services, nil +} + +func (k Kubernetes) A(zone string, state middleware.State, previousRecords []dns.RR) (records []dns.RR, err error) { + services, err := k.records(state, false) + if err != nil { + return nil, err + } + + for _, serv := range services { + ip := net.ParseIP(serv.Host) + switch { + case ip == nil: + // TODO(miek): lowercasing? Should lowercase in everything see #85 + if middleware.Name(state.Name()).Matches(dns.Fqdn(serv.Host)) { + // x CNAME x is a direct loop, don't add those + continue + } + + newRecord := serv.NewCNAME(state.QName(), serv.Host) + if len(previousRecords) > 7 { + // don't add it, and just continue + continue + } + if isDuplicateCNAME(newRecord, previousRecords) { + continue + } + + state1 := copyState(state, serv.Host, state.QType()) + nextRecords, err := k.A(zone, state1, append(previousRecords, newRecord)) + + if err == nil { + // Not only have we found something we should add the CNAME and the IP addresses. + if len(nextRecords) > 0 { + records = append(records, newRecord) + records = append(records, nextRecords...) + } + continue + } + // This means we can not complete the CNAME, try to look else where. + target := newRecord.Target + if dns.IsSubDomain(zone, target) { + // We should already have found it + continue + } + m1, e1 := k.Proxy.Lookup(state, target, state.QType()) + if e1 != nil { + continue + } + // Len(m1.Answer) > 0 here is well? + records = append(records, newRecord) + records = append(records, m1.Answer...) + continue + case ip.To4() != nil: + records = append(records, serv.NewA(state.QName(), ip.To4())) + case ip.To4() == nil: + // nodata? + } + } + return records, nil +} + +func (k Kubernetes) AAAA(zone string, state middleware.State, previousRecords []dns.RR) (records []dns.RR, err error) { + services, err := k.records(state, false) + if err != nil { + return nil, err + } + + for _, serv := range services { + ip := net.ParseIP(serv.Host) + switch { + case ip == nil: + // Try to resolve as CNAME if it's not an IP, but only if we don't create loops. + if middleware.Name(state.Name()).Matches(dns.Fqdn(serv.Host)) { + // x CNAME x is a direct loop, don't add those + continue + } + + newRecord := serv.NewCNAME(state.QName(), serv.Host) + if len(previousRecords) > 7 { + // don't add it, and just continue + continue + } + if isDuplicateCNAME(newRecord, previousRecords) { + continue + } + + state1 := copyState(state, serv.Host, state.QType()) + nextRecords, err := k.AAAA(zone, state1, append(previousRecords, newRecord)) + + if err == nil { + // Not only have we found something we should add the CNAME and the IP addresses. + if len(nextRecords) > 0 { + records = append(records, newRecord) + records = append(records, nextRecords...) + } + continue + } + // This means we can not complete the CNAME, try to look else where. + target := newRecord.Target + if dns.IsSubDomain(zone, target) { + // We should already have found it + continue + } + m1, e1 := k.Proxy.Lookup(state, target, state.QType()) + if e1 != nil { + continue + } + // Len(m1.Answer) > 0 here is well? + records = append(records, newRecord) + records = append(records, m1.Answer...) + continue + // both here again + case ip.To4() != nil: + // nada? + case ip.To4() == nil: + records = append(records, serv.NewAAAA(state.QName(), ip.To16())) + } + } + return records, nil +} + +// SRV returns SRV records from etcd. +// If the Target is not a name but an IP address, a name is created on the fly. +func (k Kubernetes) SRV(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) { + services, err := k.records(state, false) + if err != nil { + return nil, nil, err + } + + // Looping twice to get the right weight vs priority + w := make(map[int]int) + for _, serv := range services { + weight := 100 + if serv.Weight != 0 { + weight = serv.Weight + } + if _, ok := w[serv.Priority]; !ok { + w[serv.Priority] = weight + continue + } + w[serv.Priority] += weight + } + lookup := make(map[string]bool) + for _, serv := range services { + w1 := 100.0 / float64(w[serv.Priority]) + if serv.Weight == 0 { + w1 *= 100 + } else { + w1 *= float64(serv.Weight) + } + weight := uint16(math.Floor(w1)) + ip := net.ParseIP(serv.Host) + switch { + case ip == nil: + srv := serv.NewSRV(state.QName(), weight) + records = append(records, srv) + + if _, ok := lookup[srv.Target]; ok { + break + } + + lookup[srv.Target] = true + + if !dns.IsSubDomain(zone, srv.Target) { + m1, e1 := k.Proxy.Lookup(state, srv.Target, dns.TypeA) + if e1 == nil { + extra = append(extra, m1.Answer...) + } + m1, e1 = k.Proxy.Lookup(state, srv.Target, dns.TypeAAAA) + if e1 == nil { + // If we have seen CNAME's we *assume* that they are already added. + for _, a := range m1.Answer { + if _, ok := a.(*dns.CNAME); !ok { + extra = append(extra, a) + } + } + } + break + } + // Internal name, we should have some info on them, either v4 or v6 + // Clients expect a complete answer, because we are a recursor in their view. + state1 := copyState(state, srv.Target, dns.TypeA) + addr, e1 := k.A(zone, state1, nil) + if e1 == nil { + extra = append(extra, addr...) + } + // k.AAA(zone, state1, nil) as well...? + case ip.To4() != nil: + serv.Host = k.Domain(serv.Key) + srv := serv.NewSRV(state.QName(), weight) + + records = append(records, srv) + extra = append(extra, serv.NewA(srv.Target, ip.To4())) + case ip.To4() == nil: + serv.Host = k.Domain(serv.Key) + srv := serv.NewSRV(state.QName(), weight) + + records = append(records, srv) + extra = append(extra, serv.NewAAAA(srv.Target, ip.To16())) + } + } + return records, extra, nil +} + +// Returning MX records from kubernetes not implemented. +func (k Kubernetes) MX(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) { + return nil, nil, err +} + +// Returning CNAME records from kubernetes not implemented. +func (k Kubernetes) CNAME(zone string, state middleware.State) (records []dns.RR, err error) { + return nil, err +} + +// Returning TXT records from kubernetes not implemented. +func (k Kubernetes) TXT(zone string, state middleware.State) (records []dns.RR, err error) { + return nil, err +} + +func (k Kubernetes) NS(zone string, state middleware.State) (records, extra []dns.RR, err error) { + // NS record for this zone live in a special place, ns.dns.<zone>. Fake our lookup. + // only a tad bit fishy... + old := state.QName() + + state.Clear() + state.Req.Question[0].Name = "ns.dns." + zone + services, err := k.records(state, false) + if err != nil { + return nil, nil, err + } + // ... and reset + state.Req.Question[0].Name = old + + for _, serv := range services { + ip := net.ParseIP(serv.Host) + switch { + case ip == nil: + return nil, nil, fmt.Errorf("NS record must be an IP address: %s", serv.Host) + case ip.To4() != nil: + serv.Host = k.Domain(serv.Key) + records = append(records, serv.NewNS(state.QName())) + extra = append(extra, serv.NewA(serv.Host, ip.To4())) + case ip.To4() == nil: + serv.Host = k.Domain(serv.Key) + records = append(records, serv.NewNS(state.QName())) + extra = append(extra, serv.NewAAAA(serv.Host, ip.To16())) + } + } + return records, extra, nil +} + +// SOA Record returns a SOA record. +func (k Kubernetes) SOA(zone string, state middleware.State) *dns.SOA { + header := dns.RR_Header{Name: zone, Rrtype: dns.TypeSOA, Ttl: 300, Class: dns.ClassINET} + return &dns.SOA{Hdr: header, + Mbox: "hostmaster." + zone, + Ns: "ns.dns." + zone, + Serial: uint32(time.Now().Unix()), + Refresh: 7200, + Retry: 1800, + Expire: 86400, + Minttl: 60, + } +} + +// TODO(miek): DNSKEY and friends... intercepted by the DNSSEC middleware? + +func isDuplicateCNAME(r *dns.CNAME, records []dns.RR) bool { + for _, rec := range records { + if v, ok := rec.(*dns.CNAME); ok { + if v.Target == r.Target { + return true + } + } + } + return false +} + +func copyState(state middleware.State, target string, typ uint16) middleware.State { + state1 := middleware.State{W: state.W, Req: state.Req.Copy()} + state1.Req.Question[0] = dns.Question{dns.Fqdn(target), dns.ClassINET, typ} + return state1 +} diff --git a/middleware/kubernetes/msg/service.go b/middleware/kubernetes/msg/service.go new file mode 100644 index 000000000..588e7b33c --- /dev/null +++ b/middleware/kubernetes/msg/service.go @@ -0,0 +1,166 @@ +package msg + +import ( + "net" + "strings" + + "github.com/miekg/dns" +) + +// This *is* the rdata from a SRV record, but with a twist. +// Host (Target in SRV) must be a domain name, but if it looks like an IP +// address (4/6), we will treat it like an IP address. +type Service struct { + Host string `json:"host,omitempty"` + Port int `json:"port,omitempty"` + Priority int `json:"priority,omitempty"` + Weight int `json:"weight,omitempty"` + Text string `json:"text,omitempty"` + Mail bool `json:"mail,omitempty"` // Be an MX record. Priority becomes Preference. + Ttl uint32 `json:"ttl,omitempty"` + + // When a SRV record with a "Host: IP-address" is added, we synthesize + // a srv.Target domain name. Normally we convert the full Key where + // the record lives to a DNS name and use this as the srv.Target. When + // TargetStrip > 0 we strip the left most TargetStrip labels from the + // DNS name. + TargetStrip int `json:"targetstrip,omitempty"` + + // Group is used to group (or *not* to group) different services + // together. Services with an identical Group are returned in the same + // answer. + Group string `json:"group,omitempty"` + + // Etcd key where we found this service and ignored from json un-/marshalling + Key string `json:"-"` +} + +// NewSRV returns a new SRV record based on the Service. +func (s *Service) NewSRV(name string, weight uint16) *dns.SRV { + host := targetStrip(dns.Fqdn(s.Host), s.TargetStrip) + + return &dns.SRV{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeSRV, Class: dns.ClassINET, Ttl: s.Ttl}, + Priority: uint16(s.Priority), Weight: weight, Port: uint16(s.Port), Target: dns.Fqdn(host)} +} + +// NewMX returns a new MX record based on the Service. +func (s *Service) NewMX(name string) *dns.MX { + host := targetStrip(dns.Fqdn(s.Host), s.TargetStrip) + + return &dns.MX{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: s.Ttl}, + Preference: uint16(s.Priority), Mx: host} +} + +// NewA returns a new A record based on the Service. +func (s *Service) NewA(name string, ip net.IP) *dns.A { + return &dns.A{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.Ttl}, A: ip} +} + +// NewAAAA returns a new AAAA record based on the Service. +func (s *Service) NewAAAA(name string, ip net.IP) *dns.AAAA { + return &dns.AAAA{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.Ttl}, AAAA: ip} +} + +// NewCNAME returns a new CNAME record based on the Service. +func (s *Service) NewCNAME(name string, target string) *dns.CNAME { + return &dns.CNAME{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeCNAME, Class: dns.ClassINET, Ttl: s.Ttl}, Target: dns.Fqdn(target)} +} + +// NewTXT returns a new TXT record based on the Service. +func (s *Service) NewTXT(name string) *dns.TXT { + return &dns.TXT{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeTXT, Class: dns.ClassINET, Ttl: s.Ttl}, Txt: split255(s.Text)} +} + +// NewNS returns a new NS record based on the Service. +func (s *Service) NewNS(name string) *dns.NS { + host := targetStrip(dns.Fqdn(s.Host), s.TargetStrip) + return &dns.NS{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeNS, Class: dns.ClassINET, Ttl: s.Ttl}, Ns: host} +} + +// Group checks the services in sx, it looks for a Group attribute on the shortest +// keys. If there are multiple shortest keys *and* the group attribute disagrees (and +// is not empty), we don't consider it a group. +// If a group is found, only services with *that* group (or no group) will be returned. +func Group(sx []Service) []Service { + if len(sx) == 0 { + return sx + } + + // Shortest key with group attribute sets the group for this set. + group := sx[0].Group + slashes := strings.Count(sx[0].Key, "/") + length := make([]int, len(sx)) + for i, s := range sx { + x := strings.Count(s.Key, "/") + length[i] = x + if x < slashes { + if s.Group == "" { + break + } + slashes = x + group = s.Group + } + } + + if group == "" { + return sx + } + + ret := []Service{} // with slice-tricks in sx we can prolly save this allocation (TODO) + + for i, s := range sx { + if s.Group == "" { + ret = append(ret, s) + continue + } + + // Disagreement on the same level + if length[i] == slashes && s.Group != group { + return sx + } + + if s.Group == group { + ret = append(ret, s) + } + } + return ret +} + +// Split255 splits a string into 255 byte chunks. +func split255(s string) []string { + if len(s) < 255 { + return []string{s} + } + sx := []string{} + p, i := 0, 255 + for { + if i <= len(s) { + sx = append(sx, s[p:i]) + } else { + sx = append(sx, s[p:]) + break + + } + p, i = p+255, i+255 + } + + return sx +} + +// targetStrip strips "targetstrip" labels from the left side of the fully qualified name. +func targetStrip(name string, targetStrip int) string { + if targetStrip == 0 { + return name + } + + offset, end := 0, false + for i := 0; i < targetStrip; i++ { + offset, end = dns.NextLabel(name, offset) + } + if end { + // We overshot the name, use the orignal one. + offset = 0 + } + name = name[offset:] + return name +} diff --git a/middleware/kubernetes/msg/service_test.go b/middleware/kubernetes/msg/service_test.go new file mode 100644 index 000000000..0c19ba95b --- /dev/null +++ b/middleware/kubernetes/msg/service_test.go @@ -0,0 +1,125 @@ +package msg + +import "testing" + +func TestSplit255(t *testing.T) { + xs := split255("abc") + if len(xs) != 1 && xs[0] != "abc" { + t.Errorf("Failure to split abc") + } + s := "" + for i := 0; i < 255; i++ { + s += "a" + } + xs = split255(s) + if len(xs) != 1 && xs[0] != s { + t.Errorf("failure to split 255 char long string") + } + s += "b" + xs = split255(s) + if len(xs) != 2 || xs[1] != "b" { + t.Errorf("failure to split 256 char long string: %d", len(xs)) + } + for i := 0; i < 255; i++ { + s += "a" + } + xs = split255(s) + if len(xs) != 3 || xs[2] != "a" { + t.Errorf("failure to split 510 char long string: %d", len(xs)) + } +} + +func TestGroup(t *testing.T) { + // Key are in the wrong order, but for this test it does not matter. + sx := Group( + []Service{ + {Host: "127.0.0.1", Group: "g1", Key: "b/sub/dom1/skydns/test"}, + {Host: "127.0.0.2", Group: "g2", Key: "a/dom1/skydns/test"}, + }, + ) + // Expecting to return the shortest key with a Group attribute. + if len(sx) != 1 { + t.Fatalf("failure to group zeroth set: %v", sx) + } + if sx[0].Key != "a/dom1/skydns/test" { + t.Fatalf("failure to group zeroth set: %v, wrong Key", sx) + } + + // Groups disagree, so we will not do anything. + sx = Group( + []Service{ + {Host: "server1", Group: "g1", Key: "region1/skydns/test"}, + {Host: "server2", Group: "g2", Key: "region1/skydns/test"}, + }, + ) + if len(sx) != 2 { + t.Fatalf("failure to group first set: %v", sx) + } + + // Group is g1, include only the top-level one. + sx = Group( + []Service{ + {Host: "server1", Group: "g1", Key: "a/dom/region1/skydns/test"}, + {Host: "server2", Group: "g2", Key: "a/subdom/dom/region1/skydns/test"}, + }, + ) + if len(sx) != 1 { + t.Fatalf("failure to group second set: %v", sx) + } + + // Groupless services must be included. + sx = Group( + []Service{ + {Host: "server1", Group: "g1", Key: "a/dom/region1/skydns/test"}, + {Host: "server2", Group: "g2", Key: "a/subdom/dom/region1/skydns/test"}, + {Host: "server2", Group: "", Key: "b/subdom/dom/region1/skydns/test"}, + }, + ) + if len(sx) != 2 { + t.Fatalf("failure to group third set: %v", sx) + } + + // Empty group on the highest level: include that one also. + sx = Group( + []Service{ + {Host: "server1", Group: "g1", Key: "a/dom/region1/skydns/test"}, + {Host: "server1", Group: "", Key: "b/dom/region1/skydns/test"}, + {Host: "server2", Group: "g2", Key: "a/subdom/dom/region1/skydns/test"}, + }, + ) + if len(sx) != 2 { + t.Fatalf("failure to group fourth set: %v", sx) + } + + // Empty group on the highest level: include that one also, and the rest. + sx = Group( + []Service{ + {Host: "server1", Group: "g5", Key: "a/dom/region1/skydns/test"}, + {Host: "server1", Group: "", Key: "b/dom/region1/skydns/test"}, + {Host: "server2", Group: "g5", Key: "a/subdom/dom/region1/skydns/test"}, + }, + ) + if len(sx) != 3 { + t.Fatalf("failure to group fith set: %v", sx) + } + + // One group. + sx = Group( + []Service{ + {Host: "server1", Group: "g6", Key: "a/dom/region1/skydns/test"}, + }, + ) + if len(sx) != 1 { + t.Fatalf("failure to group sixth set: %v", sx) + } + + // No group, once service + sx = Group( + []Service{ + {Host: "server1", Key: "a/dom/region1/skydns/test"}, + }, + ) + if len(sx) != 1 { + t.Fatalf("failure to group seventh set: %v", sx) + } +} diff --git a/middleware/kubernetes/path.go b/middleware/kubernetes/path.go new file mode 100644 index 000000000..18c26f949 --- /dev/null +++ b/middleware/kubernetes/path.go @@ -0,0 +1,17 @@ +package kubernetes + +import ( + "strings" + + "github.com/miekg/dns" +) + +// Domain is the opposite of Path. +func (k Kubernetes) Domain(s string) string { + l := strings.Split(s, "/") + // start with 1, to strip /skydns + for i, j := 1, len(l)-1; i < j; i, j = i+1, j-1 { + l[i], l[j] = l[j], l[i] + } + return dns.Fqdn(strings.Join(l[1:len(l)-1], ".")) +} |