awsvpc.go 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328
  1. // Copyright 2015 flannel authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // +build !windows
  15. package awsvpc
  16. import (
  17. "encoding/json"
  18. "fmt"
  19. "net"
  20. "github.com/aws/aws-sdk-go/aws"
  21. "github.com/aws/aws-sdk-go/aws/awserr"
  22. "github.com/aws/aws-sdk-go/aws/ec2metadata"
  23. "github.com/aws/aws-sdk-go/aws/session"
  24. "github.com/aws/aws-sdk-go/service/ec2"
  25. log "github.com/golang/glog"
  26. "golang.org/x/net/context"
  27. "github.com/coreos/flannel/backend"
  28. "github.com/coreos/flannel/pkg/ip"
  29. "github.com/coreos/flannel/subnet"
  30. )
  31. func init() {
  32. backend.Register("aws-vpc", New)
  33. }
  34. type AwsVpcBackend struct {
  35. sm subnet.Manager
  36. extIface *backend.ExternalInterface
  37. }
  38. func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
  39. be := AwsVpcBackend{
  40. sm: sm,
  41. extIface: extIface,
  42. }
  43. return &be, nil
  44. }
  45. type backendConfig struct {
  46. RouteTableID interface{} `json:"RouteTableID"`
  47. }
  48. func (conf *backendConfig) routeTables() ([]string, error) {
  49. if table, ok := conf.RouteTableID.(string); ok {
  50. log.Info("RouteTableID configured as string: %s", table)
  51. return []string{table}, nil
  52. }
  53. if rawTables, ok := conf.RouteTableID.([]interface{}); ok {
  54. log.Info("RouteTableID configured as slice: %+v", rawTables)
  55. tables := make([]string, len(rawTables))
  56. for idx, t := range rawTables {
  57. table, ok := t.(string)
  58. if !ok {
  59. return nil, fmt.Errorf("Unexpected type in RouteTableID slice. Must be strings.")
  60. }
  61. tables[idx] = table
  62. }
  63. return tables, nil
  64. }
  65. return nil, fmt.Errorf("Unexpected RouteTableID type. Must be string or array of strings.")
  66. }
  67. func (conf *backendConfig) routeTableConfigured() bool {
  68. configured := conf.RouteTableID != nil
  69. log.Infof("Route table configured: %t", configured)
  70. return configured
  71. }
  72. func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, config *subnet.Config) (backend.Network, error) {
  73. // Parse our configuration
  74. var cfg backendConfig
  75. if len(config.Backend) > 0 {
  76. log.Info("Backend configured as: %s", string(config.Backend))
  77. if err := json.Unmarshal(config.Backend, &cfg); err != nil {
  78. return nil, fmt.Errorf("error decoding VPC backend config: %v", err)
  79. }
  80. }
  81. // Acquire the lease form subnet manager
  82. attrs := subnet.LeaseAttrs{
  83. PublicIP: ip.FromIP(be.extIface.ExtAddr),
  84. }
  85. l, err := be.sm.AcquireLease(ctx, &attrs)
  86. switch err {
  87. case nil:
  88. case context.Canceled, context.DeadlineExceeded:
  89. return nil, err
  90. default:
  91. return nil, fmt.Errorf("failed to acquire lease: %v", err)
  92. }
  93. sess, _ := session.NewSession(aws.NewConfig().WithMaxRetries(5))
  94. // Figure out this machine's EC2 instance ID and region
  95. metadataClient := ec2metadata.New(sess)
  96. region, err := metadataClient.Region()
  97. if err != nil {
  98. return nil, fmt.Errorf("error getting EC2 region name: %v", err)
  99. }
  100. sess.Config.Region = aws.String(region)
  101. instanceID, err := metadataClient.GetMetadata("instance-id")
  102. if err != nil {
  103. return nil, fmt.Errorf("error getting EC2 instance ID: %v", err)
  104. }
  105. ec2c := ec2.New(sess)
  106. // Find ENI which contains the external network interface IP address
  107. eni, err := be.findENI(instanceID, ec2c)
  108. if err != nil || eni == nil {
  109. return nil, fmt.Errorf("unable to find ENI that matches the %s IP address. %s\n", be.extIface.IfaceAddr, err)
  110. }
  111. // Try to disable SourceDestCheck on the main network interface
  112. if err := be.disableSrcDestCheck(eni.NetworkInterfaceId, ec2c); err != nil {
  113. log.Warningf("failed to disable SourceDestCheck on %s: %s.\n", *eni.NetworkInterfaceId, err)
  114. }
  115. if !cfg.routeTableConfigured() {
  116. if cfg.RouteTableID, err = be.detectRouteTableID(eni, ec2c); err != nil {
  117. return nil, err
  118. }
  119. log.Infof("Found route table %s.\n", cfg.RouteTableID)
  120. }
  121. networkConfig, err := be.sm.GetNetworkConfig(ctx)
  122. if err != nil {
  123. log.Errorf("Error fetching network config: %v", err)
  124. }
  125. tables, err := cfg.routeTables()
  126. if err != nil {
  127. return nil, err
  128. }
  129. for _, routeTableID := range tables {
  130. err = be.cleanupBlackholeRoutes(routeTableID, networkConfig.Network, ec2c)
  131. if err != nil {
  132. log.Errorf("Error cleaning up blackhole routes: %v", err)
  133. }
  134. matchingRouteFound, err := be.checkMatchingRoutes(routeTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c)
  135. if err != nil {
  136. log.Errorf("Error describing route tables: %v", err)
  137. }
  138. if !matchingRouteFound {
  139. cidrBlock := l.Subnet.String()
  140. deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &routeTableID, DestinationCidrBlock: &cidrBlock}
  141. if _, err := ec2c.DeleteRoute(deleteRouteInput); err != nil {
  142. if ec2err, ok := err.(awserr.Error); !ok || ec2err.Code() != "InvalidRoute.NotFound" {
  143. // an error other than the route not already existing occurred
  144. return nil, fmt.Errorf("error deleting existing route for %s: %v", l.Subnet.String(), err)
  145. }
  146. }
  147. // Add the route for this machine's subnet
  148. if err := be.createRoute(routeTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c); err != nil {
  149. return nil, fmt.Errorf("unable to add route %s: %v", l.Subnet.String(), err)
  150. }
  151. }
  152. }
  153. return &backend.SimpleNetwork{
  154. SubnetLease: l,
  155. ExtIface: be.extIface,
  156. }, nil
  157. }
  158. func (be *AwsVpcBackend) cleanupBlackholeRoutes(routeTableID string, network ip.IP4Net, ec2c *ec2.EC2) error {
  159. filter := newFilter()
  160. filter.Add("route.state", "blackhole")
  161. input := ec2.DescribeRouteTablesInput{Filters: filter, RouteTableIds: []*string{&routeTableID}}
  162. resp, err := ec2c.DescribeRouteTables(&input)
  163. if err != nil {
  164. return err
  165. }
  166. for _, routeTable := range resp.RouteTables {
  167. for _, route := range routeTable.Routes {
  168. if *route.State == "blackhole" && route.DestinationCidrBlock != nil {
  169. _, subnet, err := net.ParseCIDR(*route.DestinationCidrBlock)
  170. if err == nil && network.Contains(ip.FromIP(subnet.IP)) {
  171. log.Info("Removing blackhole route: ", *route.DestinationCidrBlock)
  172. deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &routeTableID, DestinationCidrBlock: route.DestinationCidrBlock}
  173. if _, err := ec2c.DeleteRoute(deleteRouteInput); err != nil {
  174. if ec2err, ok := err.(awserr.Error); !ok || ec2err.Code() != "InvalidRoute.NotFound" {
  175. // an error other than the route not already existing occurred
  176. return err
  177. }
  178. }
  179. }
  180. }
  181. }
  182. }
  183. return nil
  184. }
  185. func (be *AwsVpcBackend) checkMatchingRoutes(routeTableID, subnet string, eniID *string, ec2c *ec2.EC2) (bool, error) {
  186. matchingRouteFound := false
  187. filter := newFilter()
  188. filter.Add("route.destination-cidr-block", subnet)
  189. filter.Add("route.state", "active")
  190. input := ec2.DescribeRouteTablesInput{Filters: filter, RouteTableIds: []*string{&routeTableID}}
  191. resp, err := ec2c.DescribeRouteTables(&input)
  192. if err != nil {
  193. return matchingRouteFound, err
  194. }
  195. for _, routeTable := range resp.RouteTables {
  196. for _, route := range routeTable.Routes {
  197. if route.DestinationCidrBlock != nil && subnet == *route.DestinationCidrBlock &&
  198. *route.State == "active" && route.NetworkInterfaceId == eniID {
  199. matchingRouteFound = true
  200. break
  201. }
  202. }
  203. }
  204. return matchingRouteFound, nil
  205. }
  206. func (be *AwsVpcBackend) createRoute(routeTableID, subnet string, eniID *string, ec2c *ec2.EC2) error {
  207. route := &ec2.CreateRouteInput{
  208. RouteTableId: &routeTableID,
  209. NetworkInterfaceId: eniID,
  210. DestinationCidrBlock: &subnet,
  211. }
  212. if _, err := ec2c.CreateRoute(route); err != nil {
  213. return err
  214. }
  215. log.Infof("Route added to table %s: %s - %s.\n", routeTableID, subnet, *eniID)
  216. return nil
  217. }
  218. func (be *AwsVpcBackend) disableSrcDestCheck(eniID *string, ec2c *ec2.EC2) error {
  219. attr := &ec2.ModifyNetworkInterfaceAttributeInput{
  220. NetworkInterfaceId: eniID,
  221. SourceDestCheck: &ec2.AttributeBooleanValue{Value: aws.Bool(false)},
  222. }
  223. _, err := ec2c.ModifyNetworkInterfaceAttribute(attr)
  224. return err
  225. }
  226. // detectRouteTableID detect the routing table that is associated with the ENI,
  227. // subnet can be implicitly associated with the main routing table
  228. func (be *AwsVpcBackend) detectRouteTableID(eni *ec2.InstanceNetworkInterface, ec2c *ec2.EC2) (string, error) {
  229. subnetID := eni.SubnetId
  230. vpcID := eni.VpcId
  231. filter := newFilter()
  232. filter.Add("association.subnet-id", *subnetID)
  233. routeTablesInput := &ec2.DescribeRouteTablesInput{
  234. Filters: filter,
  235. }
  236. res, err := ec2c.DescribeRouteTables(routeTablesInput)
  237. if err != nil {
  238. return "", fmt.Errorf("error describing routeTables for subnetID %s: %v", *subnetID, err)
  239. }
  240. if len(res.RouteTables) != 0 {
  241. return *res.RouteTables[0].RouteTableId, nil
  242. }
  243. filter = newFilter()
  244. filter.Add("association.main", "true")
  245. filter.Add("vpc-id", *vpcID)
  246. routeTablesInput = &ec2.DescribeRouteTablesInput{
  247. Filters: filter,
  248. }
  249. res, err = ec2c.DescribeRouteTables(routeTablesInput)
  250. if err != nil {
  251. log.Info("error describing route tables: ", err)
  252. }
  253. if len(res.RouteTables) == 0 {
  254. return "", fmt.Errorf("main route table not found")
  255. }
  256. return *res.RouteTables[0].RouteTableId, nil
  257. }
  258. func (be *AwsVpcBackend) findENI(instanceID string, ec2c *ec2.EC2) (*ec2.InstanceNetworkInterface, error) {
  259. instance, err := ec2c.DescribeInstances(&ec2.DescribeInstancesInput{
  260. InstanceIds: []*string{aws.String(instanceID)}},
  261. )
  262. if err != nil {
  263. return nil, err
  264. }
  265. for _, n := range instance.Reservations[0].Instances[0].NetworkInterfaces {
  266. for _, a := range n.PrivateIpAddresses {
  267. if *a.PrivateIpAddress == be.extIface.IfaceAddr.String() {
  268. log.Infof("Found %s that has %s IP address.\n", *n.NetworkInterfaceId, be.extIface.IfaceAddr)
  269. return n, nil
  270. }
  271. }
  272. }
  273. return nil, err
  274. }