cf-charmers team mailing list archive
-
cf-charmers team
-
Mailing list archive
-
Message #00646
[Merge] lp:~johnsca/charms/trusty/cloudfoundry/catchup into lp:~cf-charmers/charms/trusty/cloudfoundry/trunk
Cory Johns has proposed merging lp:~johnsca/charms/trusty/cloudfoundry/catchup into lp:~cf-charmers/charms/trusty/cloudfoundry/trunk.
Requested reviews:
Cloud Foundry Charmers (cf-charmers)
For more details, see:
https://code.launchpad.net/~johnsca/charms/trusty/cloudfoundry/catchup/+merge/244226
Changes to support 190. I haven't run CATS on it, but pushing an app and logging works.
--
Your team Cloud Foundry Charmers is requested to review the proposed merge of lp:~johnsca/charms/trusty/cloudfoundry/catchup into lp:~cf-charmers/charms/trusty/cloudfoundry/trunk.
=== modified file 'charmgen/generator.py'
--- charmgen/generator.py 2014-11-07 21:50:48 +0000
+++ charmgen/generator.py 2014-12-09 21:36:57 +0000
@@ -227,6 +227,10 @@
if zone:
driver = zone[0]
for follower in zone[1:]:
+ if follower not in services:
+ # allow dangling references in placement to handle
+ # removed services
+ continue
with_references.add(driver)
if 'to' not in services[follower]:
services[follower]['to'] = [driver]
=== modified file 'charmgen/placements.yaml'
--- charmgen/placements.yaml 2014-11-11 03:52:11 +0000
+++ charmgen/placements.yaml 2014-12-09 21:36:57 +0000
@@ -5,7 +5,7 @@
b: [router, nats, nats-sf]
c: [login, uaa, mysql]
d: [dea, hm, etcd]
- e: [loggregator, loggregator-trafficcontrol]
+ e: [loggregator-trafficcontrol, loggregator, doppler]
f: [haproxy, collector]
local:
__default__:
=== modified file 'cloudfoundry/contexts.py'
--- cloudfoundry/contexts.py 2014-11-11 04:02:53 +0000
+++ cloudfoundry/contexts.py 2014-12-09 21:36:57 +0000
@@ -226,7 +226,7 @@
class LoginRelation(RelationContext):
name = 'login'
- interface = 'http'
+ interface = 'login'
required_keys = []
port = 8080
@@ -239,6 +239,8 @@
data = self[self.name][0]
return {
'login.port': data['port'],
+ 'login.saml.socket.connectionManagerTimeout': 10000,
+ 'login.saml.socket.soTimeout': 10000,
}
@@ -250,6 +252,11 @@
def erb_mapping(self):
return {
'dea_next.directory_server_protocol': 'http',
+ # these are the default values, but since they're used in hm and
+ # cc and those specs don't give the default value (only dea_next),
+ # we have to manually specify them here
+ 'dea_next.advertise_interval_in_seconds': 5,
+ 'dea_next.heartbeat_interval_in_seconds': 10,
}
@@ -296,8 +303,6 @@
'traffic_controller.host': data[0]['host'],
'traffic_controller.incoming_port': data[0]['port'],
'traffic_controller.outgoing_port': data[0]['outgoing_port'],
- 'logger_endpoint.use_ssl': False, # TODO: support SSL option
- 'logger_endpoint.port': 80, # default is 443
}
@@ -328,6 +333,48 @@
}
+class DopplerRelation(RelationContext):
+ name = 'doppler'
+ interface = 'doppler'
+ required_keys = ['address', 'incoming_port', 'outgoing_port',
+ 'dropsonde_port', 'doppler_endpoint_secret']
+ incoming_port = 3457
+ outgoing_port = 8083
+ dropsonde_port = 3460
+
+ def get_shared_secrets(self):
+ secret_context = StoredContext('doppler-secrets.yml', {
+ 'doppler_endpoint_secret': host.pwgen(20),
+ })
+ return secret_context
+
+ def provide_data(self):
+ secrets = self.get_shared_secrets()
+ return {
+ 'address': hookenv.unit_get('private-address').encode('utf-8'),
+ 'incoming_port': self.incoming_port,
+ 'outgoing_port': self.outgoing_port,
+ 'dropsonde_port': self.dropsonde_port,
+ 'doppler_endpoint_secret': secrets['doppler_endpoint_secret']
+ }
+
+ def erb_mapping(self):
+ data = self[self.name]
+ return {
+ 'doppler.zone': 'z1',
+ 'doppler.blacklisted_syslog_ranges': [],
+ 'doppler.incoming_port': data[0]['incoming_port'],
+ 'doppler.outgoing_port': data[0]['outgoing_port'],
+ 'doppler.dropsonde_incoming_port': data[0]['dropsonde_port'],
+ 'doppler_endpoint.shared_secret': data[0]['doppler_endpoint_secret'],
+ 'loggregator.doppler_port': data[0]['outgoing_port'],
+ 'loggregator.incoming_port': data[0]['incoming_port'],
+ 'loggregator.dropsonde_incoming_port': data[0]['dropsonde_port'],
+ 'loggregator.dropsonde_outgoing_port': data[0]['outgoing_port'],
+ 'loggregator_endpoint.shared_secret': data[0]['doppler_endpoint_secret'],
+ }
+
+
class EtcdRelation(RelationContext):
name = 'etcd'
interface = 'etcd'
@@ -343,13 +390,15 @@
class CloudControllerRelation(RelationContext):
name = 'cc'
interface = 'controller'
- required_keys = ['hostname', 'port', 'user', 'password', 'db_encryption_key']
+ required_keys = ['hostname', 'port', 'user', 'password',
+ 'internal_api_password', 'db_encryption_key']
def get_credentials(self):
return StoredContext('api_credentials.yml', {
'user': host.pwgen(7),
'password': host.pwgen(7),
'db_encryption_key': host.pwgen(7),
+ 'internal_api_password': host.pwgen(7),
})
def provide_data(self):
@@ -357,6 +406,7 @@
return {
'user': creds['user'],
'password': creds['password'],
+ 'internal_api_password': creds['internal_api_password'],
'db_encryption_key': creds['db_encryption_key'],
'hostname': hookenv.unit_get('private-address').encode('utf-8'),
'port': 9022,
@@ -369,6 +419,7 @@
'cc.srv_api_uri': 'http://{}:{}'.format(data['hostname'], data['port']),
'cc.bulk_api_user': data['user'],
'cc.bulk_api_password': data['password'],
+ 'cc.internal_api_password': data['internal_api_password'],
'cc.staging_upload_user': 'ignored', # FIXME: We need a staging cache set up
'cc.staging_upload_password': 'ignored',
'cc.db_encryption_key': data['db_encryption_key'],
@@ -439,7 +490,7 @@
class RouterRelation(RelationContext):
name = 'router'
- interface = 'http'
+ interface = 'router'
required_keys = ['address']
port = 8000
varz_port = 8084 # not currently used
@@ -545,6 +596,8 @@
'app_domains': [d['domain'] for d in self[self.name]],
'system_domain': domain, # TODO: These should probably be config options
'system_domain_organization': 'juju-org',
+ 'logger_endpoint.use_ssl': False, # TODO: support SSL option
+ 'logger_endpoint.port': 80, # default is 443
}
=== modified file 'cloudfoundry/releases.py'
--- cloudfoundry/releases.py 2014-11-05 18:22:07 +0000
+++ cloudfoundry/releases.py 2014-12-09 21:36:57 +0000
@@ -8,8 +8,6 @@
('uaa-v1', 'uaa'),
('login-v1', 'login'),
('nats-stream-forwarder-v1', 'nats-sf'),
- ('loggregator-v1', 'loggregator'),
- ('hm9000-v1', 'hm'),
('haproxy-v1', 'haproxy'),
('collector-v1', 'collector'),
@@ -25,7 +23,6 @@
('mysql:db', 'cc:db'),
('mysql:db', 'uaa:db'),
('etcd:client', 'hm:etcd'),
- ('etcd:client', 'loggregator:etcd'),
('etcd:client', 'loggregator-trafficcontrol:etcd'),
]
@@ -34,6 +31,38 @@
RELEASES = [
{
+ "releases": (190, 190),
+ "topology": {
+ "services": COMMON_SERVICES + [
+ ('router-v3', 'router'),
+ ('cloud-controller-v3', 'cc'),
+ ('cloud-controller-clock-v3', 'cc-clock'),
+ ('cloud-controller-worker-v3', 'cc-worker'),
+ ('dea-v3', 'dea'),
+ ('loggregator-trafficcontroller-v3', 'loggregator-trafficcontrol'),
+ ('hm9000-v2', 'hm'),
+ ('doppler-v1', 'doppler'),
+ ],
+ "relations": COMMON_RELATIONS + [
+ ('etcd:client', 'cc:etcd'),
+ ('etcd:client', 'cc-worker:etcd'),
+ ('etcd:client', 'cc-clock:etcd'),
+ ('etcd:client', 'router:etcd'),
+ ('etcd:client', 'dea:etcd'),
+ ('etcd:client', 'doppler:etcd'),
+ ],
+ "expose": ['haproxy'],
+ "constraints": {
+ "__default__": "arch=amd64 instance-type=m3.medium",
+ "cc": "arch=amd64 root-disk=12G mem=12G",
+ "cc-worker": "arch=amd64 root-disk=10G",
+ "cc-clock": "arch=amd64 root-disk=10G",
+ "dea": "arch=amd64 mem=5G",
+ },
+ },
+ "upgrades": COMMON_UPGRADES
+ },
+ {
"releases": (177, 180),
"topology": {
"services": COMMON_SERVICES + [
@@ -42,7 +71,9 @@
('cloud-controller-clock-v2', 'cc-clock'),
('cloud-controller-worker-v2', 'cc-worker'),
('dea-v2', 'dea'),
+ ('loggregator-v1', 'loggregator'),
('loggregator-trafficcontroller-v2', 'loggregator-trafficcontrol'),
+ ('hm9000-v1', 'hm'),
],
"relations": COMMON_RELATIONS + [
('etcd:client', 'cc:etcd'),
@@ -50,6 +81,7 @@
('etcd:client', 'cc-clock:etcd'),
('etcd:client', 'router:etcd'),
('etcd:client', 'dea:etcd'),
+ ('etcd:client', 'loggregator:etcd'),
],
"expose": ['haproxy'],
"constraints": {
@@ -71,9 +103,13 @@
('cloud-controller-clock-v1', 'cc-clock'),
('cloud-controller-worker-v1', 'cc-worker'),
('dea-v1', 'dea'),
+ ('loggregator-v1', 'loggregator'),
('loggregator-trafficcontroller-v1', 'loggregator-trafficcontrol'),
- ],
- "relations": COMMON_RELATIONS,
+ ('hm9000-v1', 'hm'),
+ ],
+ "relations": COMMON_RELATIONS + [
+ ('etcd:client', 'loggregator:etcd'),
+ ],
"expose": ['haproxy'],
"constraints": {
"__default__": "arch=amd64",
=== modified file 'cloudfoundry/services.py'
--- cloudfoundry/services.py 2014-11-07 21:18:42 +0000
+++ cloudfoundry/services.py 2014-12-09 21:36:57 +0000
@@ -49,6 +49,27 @@
},
+ 'cloud-controller-clock-v3': {
+ 'summary': "A shared clock",
+ 'description': '',
+ 'jobs': [
+ {'job_name': 'cloud_controller_clock',
+ 'mapping': {'ccdb': mapper.ccdb},
+ 'provided_data': [],
+ 'required_data': [contexts.NatsRelation,
+ contexts.CloudControllerRelation,
+ contexts.UAARelation,
+ contexts.CloudControllerDBRelation],
+ },
+ {'job_name': 'metron_agent',
+ 'required_data': [contexts.LTCRelation,
+ contexts.NatsRelation,
+ contexts.DopplerRelation,
+ contexts.EtcdRelation]},
+ ],
+
+ },
+
'cloud-controller-v1': {
'summary': 'CF Cloud Controller, the brains of the operation',
'description': '',
@@ -89,6 +110,29 @@
]
},
+ 'cloud-controller-v3': {
+ 'summary': 'CF Cloud Controller, the brains of the operation',
+ 'description': '',
+ 'jobs': [
+ {'job_name': 'cloud_controller_ng',
+ 'mapping': {'db': mapper.ccdb},
+ 'provided_data': [contexts.CloudControllerRelation,
+ contexts.CloudControllerDBRelation],
+ 'required_data': [contexts.NatsRelation,
+ contexts.MysqlRelation,
+ contexts.UAARelation,
+ contexts.DEARelation,
+ contexts.CloudControllerRelation.remote_view,
+ ],
+ },
+ {'job_name': 'metron_agent',
+ 'required_data': [contexts.LTCRelation,
+ contexts.NatsRelation,
+ contexts.DopplerRelation,
+ contexts.EtcdRelation]},
+ ]
+ },
+
'cloud-controller-worker-v1': {
'summary': "Worker for cc",
'description': '',
@@ -128,6 +172,27 @@
]
},
+ 'cloud-controller-worker-v3': {
+ 'summary': "Worker for cc",
+ 'description': '',
+ 'jobs': [
+ {'job_name': 'cloud_controller_worker',
+ 'mapping': {'ccdb': mapper.ccdb},
+ 'provided_data': [],
+ 'required_data': [contexts.NatsRelation,
+ contexts.UAARelation,
+ contexts.CloudControllerRelation,
+ contexts.CloudControllerDBRelation,
+ ],
+ },
+ {'job_name': 'metron_agent',
+ 'required_data': [contexts.LTCRelation,
+ contexts.NatsRelation,
+ contexts.DopplerRelation,
+ contexts.EtcdRelation]},
+ ]
+ },
+
'dea-v1': {
'summary': 'DEA runs CF apps in containers',
'description': '',
@@ -199,6 +264,47 @@
},
+
+ 'dea-v3': {
+ 'summary': 'DEA runs CF apps in containers',
+ 'description': '',
+ 'jobs': [
+ {
+ 'job_name': 'dea_next',
+ 'mapping': {},
+ 'install': [
+ utils.install_linux_image_extra,
+ utils.apt_install(['quota']),
+ utils.modprobe(['quota_v1', 'quota_v2'])
+ ],
+ 'provided_data': [contexts.DEARelation],
+ 'required_data': [
+ contexts.NatsRelation,
+ contexts.LTCRelation,
+ contexts.DEARelation.remote_view,
+ contexts.RouterRelation,
+ ],
+ 'data_ready': [
+ #tasks.enable_swapaccounting
+ tasks.patch_dea
+ ]
+ },
+ {
+ 'job_name': 'dea_logging_agent',
+ 'mapping': {},
+ 'required_data': [contexts.NatsRelation,
+ contexts.LTCRelation,
+ contexts.EtcdRelation]
+ },
+ {'job_name': 'metron_agent',
+ 'required_data': [contexts.LTCRelation,
+ contexts.NatsRelation,
+ contexts.DopplerRelation,
+ contexts.EtcdRelation]},
+ ]
+
+ },
+
'nats-v1': {
'service': 'nats',
'summary': 'NATS message bus for CF',
@@ -259,6 +365,25 @@
},
+ 'router-v3': {
+ 'service': 'router',
+ 'summary': 'CF Router',
+ 'jobs': [
+ {'job_name': 'gorouter',
+ 'ports': [contexts.RouterRelation.port],
+ 'mapping': {},
+ 'provided_data': [contexts.RouterRelation],
+ 'required_data': [contexts.NatsRelation,
+ contexts.RouterRelation.remote_view]},
+ {'job_name': 'metron_agent',
+ 'required_data': [contexts.LTCRelation,
+ contexts.NatsRelation,
+ contexts.DopplerRelation,
+ contexts.EtcdRelation]},
+ ],
+
+ },
+
'uaa-v1': {
'service': 'uaa',
'summary': 'CF Oauth2 for identity management service',
@@ -303,6 +428,21 @@
}]
},
+ 'doppler-v1': {
+ 'service': 'doppler',
+ 'summary': 'successor of loggregator',
+ 'description': 'Successor of loggregator.',
+ 'jobs': [{
+ 'job_name': 'doppler',
+ 'mapping': {},
+ 'provided_data': [contexts.DopplerRelation],
+ 'required_data': [contexts.NatsRelation,
+ contexts.EtcdRelation,
+ contexts.LTCRelation,
+ contexts.DopplerRelation.remote_view]
+ }]
+ },
+
'loggregator-trafficcontroller-v1': {
'service': 'loggregator-trafficcontroller',
'summary': 'loggregator-trafficcontroller',
@@ -337,6 +477,23 @@
]
},
+ 'loggregator-trafficcontroller-v3': {
+ 'service': 'loggregator-trafficcontroller',
+ 'summary': 'loggregator-trafficcontroller',
+ 'description': '',
+ 'jobs': [
+ {'job_name': 'loggregator_trafficcontroller',
+ 'ports': [contexts.LTCRelation.outgoing_port],
+ 'mapping': {},
+ 'provided_data': [contexts.LTCRelation],
+ 'required_data': [contexts.DopplerRelation,
+ contexts.LTCRelation.remote_view,
+ contexts.NatsRelation,
+ contexts.CloudControllerRelation,
+ contexts.EtcdRelation]},
+ ]
+ },
+
'hm9000-v1': {
'service': 'hm9000',
'summary': 'health monitor',
@@ -351,6 +508,21 @@
}]
},
+ 'hm9000-v2': {
+ 'service': 'hm9000',
+ 'summary': 'health monitor',
+ 'description': '',
+ 'jobs': [{
+ 'job_name': 'hm9000',
+ 'mapping': {},
+ 'provided_data': [],
+ 'required_data': [contexts.NatsRelation,
+ contexts.CloudControllerRelation,
+ contexts.EtcdRelation,
+ contexts.DEARelation]
+ }]
+ },
+
'haproxy-v1': {
'service': 'haproxy',
'summary': 'loadbalance the routers',
=== modified file 'reconciler/app.py'
--- reconciler/app.py 2014-11-20 16:09:27 +0000
+++ reconciler/app.py 2014-12-09 21:36:57 +0000
@@ -135,6 +135,8 @@
service['health'] = 'warn'
else:
service['health'] = 'pass'
+ if service_name == 'cloudfoundry' and db.error: # XXX don't hard-code
+ service['health'] = 'fail'
def get_current_state():
=== modified file 'reconciler/strategy.py'
--- reconciler/strategy.py 2014-11-11 18:22:53 +0000
+++ reconciler/strategy.py 2014-12-09 21:36:57 +0000
@@ -28,6 +28,7 @@
self.strategy = Strategy(self.env)
self.history = []
self.exec_lock = threading.Lock()
+ self.error = False
def reset(self):
self.expected = {}
@@ -106,8 +107,12 @@
return []
# Service Deltas
- self.strategy.extend(self.build_services())
- self.strategy.extend(self.build_relations())
+ try:
+ self.strategy.extend(self.build_services())
+ self.strategy.extend(self.build_relations())
+ except Exception as e:
+ self.error = True
+ logging.error('Error building strategy: %s', e)
def _changed(self):
previous = hashlib.md5(json.dumps(self.previous or {}, sort_keys=True))
@@ -205,6 +210,10 @@
self.execute_strategy)
else:
self._reset_strategy()
+ self.error = False
+ except Exception as e:
+ self.error = True
+ logging.error('Error executing strategy: %s', e)
finally:
self.exec_lock.release()
Follow ups
