cm_api.endpoints.clusters

78 _ATTRIBUTES = { 79 'name' : None, 80 'displayName' : None, 81 'clusterUrl' : None, 82 'version' : None, 83 'fullVersion' : None, 84 'hostsUrl' : ROAttr(), 85 'maintenanceMode' : ROAttr(), 86 'maintenanceOwners' : ROAttr(), 87 'entityStatus' : ROAttr(), 88 'uuid' : ROAttr(), 89 } 90

91 - def __init__(self, resource_root, name=None, version=None, fullVersion=None):

92 BaseApiObject.init(self, resource_root, locals())

93

94 - def __str__(self):

95 return "<ApiCluster>: %s; version: %s" % (self.name, self.version)

96

97 - def _path(self):

98 return "%s/%s" % (CLUSTERS_PATH, self.name)

99

100 - def _put_cluster(self, dic, params=None):

101 """Change cluster attributes""" 102 cluster = self._put('', ApiCluster, data=dic, params=params) 103 self._update(cluster) 104 return self

105

106 - def get_service_types(self):

107 """ 108 Get all service types supported by this cluster. 109 110 @return: A list of service types (strings) 111 """ 112 resp = self._get_resource_root().get(self._path() + '/serviceTypes') 113 return resp[ApiList.LIST_KEY]

114

115 - def get_commands(self, view=None):

116 """ 117 Retrieve a list of running commands for this cluster. 118 119 @param view: View to materialize ('full' or 'summary') 120 @return: A list of running commands. 121 """ 122 return self._get("commands", ApiCommand, True, 123 params = view and dict(view=view) or None)

124

125 - def rename(self, newname):

126 """ 127 Rename a cluster. 128 129 @param newname: New cluster name 130 @return: An ApiCluster object 131 @since: API v2 132 """ 133 dic = self.to_json_dict() 134 if self._get_resource_root().version < 6: 135 dic['name'] = newname 136 else: 137 dic['displayName'] = newname 138 return self._put_cluster(dic)

139

140 - def update_cdh_version(self, new_cdh_version):

141 """ 142 Manually set the CDH version. 143 144 @param new_cdh_version: New CDH version, e.g. 4.5.1 145 @return: An ApiCluster object 146 @since: API v6 147 """ 148 dic = self.to_json_dict() 149 dic['fullVersion'] = new_cdh_version 150 return self._put_cluster(dic)

151

152 - def create_service(self, name, service_type):

153 """ 154 Create a service. 155 156 @param name: Service name 157 @param service_type: Service type 158 @return: An ApiService object 159 """ 160 return services.create_service(self._get_resource_root(), name, 161 service_type, self.name)

162

163 - def delete_service(self, name):

164 """ 165 Delete a service by name. 166 167 @param name: Service name 168 @return: The deleted ApiService object 169 """ 170 return services.delete_service(self._get_resource_root(), name, self.name)

171

172 - def get_service(self, name):

173 """ 174 Lookup a service by name. 175 176 @param name: Service name 177 @return: An ApiService object 178 """ 179 return services.get_service(self._get_resource_root(), name, self.name)

180

181 - def get_all_services(self, view = None):

182 """ 183 Get all services in this cluster. 184 185 @return: A list of ApiService objects. 186 """ 187 return services.get_all_services(self._get_resource_root(), self.name, view)

188

189 - def get_parcel(self, product, version):

190 """ 191 Lookup a parcel by product and version. 192 193 @param product: the product name 194 @param version: the product version 195 @return: An ApiParcel object 196 """ 197 return parcels.get_parcel(self._get_resource_root(), product, version, self.name)

198

199 - def get_all_parcels(self, view = None):

200 """ 201 Get all parcels in this cluster. 202 203 @return: A list of ApiParcel objects. 204 """ 205 return parcels.get_all_parcels(self._get_resource_root(), self.name, view)

206

207 - def list_hosts(self):

208 """ 209 Lists all the hosts that are associated with this cluster. 210 211 @return: A list of ApiHostRef objects of the hosts in the cluster. 212 @since: API v3 213 """ 214 return self._get("hosts", ApiHostRef, True, api_version=3)

215

216 - def remove_host(self, hostId):

217 """ 218 Removes the association of the host with the cluster. 219 220 @return: A ApiHostRef of the host that was removed. 221 @since: API v3 222 """ 223 return self._delete("hosts/" + hostId, ApiHostRef, api_version=3)

224

225 - def remove_all_hosts(self):

226 """ 227 Removes the association of all the hosts with the cluster. 228 229 @return: A list of ApiHostRef objects of the hosts that were removed. 230 @since: API v3 231 """ 232 return self._delete("hosts", ApiHostRef, True, api_version=3)

233

234 - def add_hosts(self, hostIds):

235 """ 236 Adds a host to the cluster. 237 238 @param hostIds: List of IDs of hosts to add to cluster. 239 @return: A list of ApiHostRef objects of the new 240 hosts that were added to the cluster 241 @since: API v3 242 """ 243 hostRefList = [ApiHostRef(self._get_resource_root(), x) for x in hostIds] 244 return self._post("hosts", ApiHostRef, True, data=hostRefList, 245 api_version=3)

246

247 - def start(self):

248 """ 249 Start all services in a cluster, respecting dependencies. 250 251 @return: Reference to the submitted command. 252 """ 253 return self._cmd('start')

254

255 - def stop(self):

256 """ 257 Stop all services in a cluster, respecting dependencies. 258 259 @return: Reference to the submitted command. 260 """ 261 return self._cmd('stop')

262

263 - def restart(self, restart_only_stale_services=None, 264 redeploy_client_configuration=None, 265 restart_service_names=None):

266 """ 267 Restart all services in the cluster. 268 Services are restarted in the appropriate order given their dependencies. 269 270 @param restart_only_stale_services: Only restart services that have stale 271 configuration and their dependent 272 services. Default is False. 273 @param redeploy_client_configuration: Re-deploy client configuration for 274 all services in the cluster. Default 275 is False. 276 @param restart_service_names: Only restart services that are specified and their dependent services. 277 Available since API v11. 278 @since API v6 279 280 @return: Reference to the submitted command. 281 """ 282 if self._get_resource_root().version < 6: 283 return self._cmd('restart') 284 else: 285 args = dict() 286 args['restartOnlyStaleServices'] = restart_only_stale_services 287 args['redeployClientConfiguration'] = redeploy_client_configuration 288 if self._get_resource_root().version >= 11: 289 args['restartServiceNames'] = restart_service_names 290 return self._cmd('restart', data=args, api_version=6)

291

292 - def deploy_client_config(self):

293 """ 294 Deploys Service client configuration to the hosts on the cluster. 295 296 @return: Reference to the submitted command. 297 @since: API v2 298 """ 299 return self._cmd('deployClientConfig')

300

301 - def deploy_cluster_client_config(self, hostIds=[]):

302 """ 303 Deploys Cluster client configuration (Kerberos configuration) to the 304 hosts on the cluster. Any hosts that are decommissioned or have running 305 roles will be skipped. 306 307 @param hostIds: hostIds of hosts to deploy to. If empty, deploys to all 308 hosts in the cluster. 309 @return: Reference to the submitted command. 310 @since: API v7 311 """ 312 return self._cmd('deployClusterClientConfig', data=hostIds, 313 api_version=7)

314

315 - def upgrade_services(self):

316 """ 317 This command is no longer recommended with API v6 onwards. It simply does 318 not work when parcels are used, and even with packages it may fail due to 319 a race. Use upgrade_cdh instead. 320 321 Upgrades the services in the cluster to CDH5 version. 322 This command requires that the CDH packages in the hosts used by the 323 cluster be upgraded to CDH5 before this command is issued. Once issued, 324 this command will stop all running services before proceeding. 325 326 If parcels are used instead of CDH system packages then the following 327 steps need to happen in order: 328 1. Stop all services manually 329 2. Activate parcel 330 3. Run this upgrade command 331 332 The command will upgrade the services and their configuration to the 333 version available in the CDH5 distribution. 334 335 @return: Reference to the submitted command. 336 @deprecated: since API v6 337 """ 338 return self._cmd('upgradeServices')

339

340 - def enter_maintenance_mode(self):

341 """ 342 Put the cluster in maintenance mode. 343 344 @return: Reference to the completed command. 345 @since: API v2 346 """ 347 cmd = self._cmd('enterMaintenanceMode') 348 if cmd.success: 349 self._update(get_cluster(self._get_resource_root(), self.name)) 350 return cmd

351

352 - def exit_maintenance_mode(self):

353 """ 354 Take the cluster out of maintenance mode. 355 356 @return: Reference to the completed command. 357 @since: API v2 358 """ 359 cmd = self._cmd('exitMaintenanceMode') 360 if cmd.success: 361 self._update(get_cluster(self._get_resource_root(), self.name)) 362 return cmd

363

364 - def get_all_host_templates(self):

365 """ 366 Retrieves all host templates in the cluster. 367 @return: ApiList of ApiHostTemplate objects. 368 """ 369 return host_templates.get_all_host_templates(self._get_resource_root(), self.name)

370

371 - def get_host_template(self, name):

372 """ 373 Retrieves a host templates by name. 374 @param name: Host template name. 375 @return: An ApiHostTemplate object. 376 """ 377 return host_templates.get_host_template(self._get_resource_root(), name, self.name)

378

379 - def create_host_template(self, name):

380 """ 381 Creates a host template. 382 @param name: Name of the host template to create. 383 @return: An ApiHostTemplate object. 384 """ 385 return host_templates.create_host_template(self._get_resource_root(), name, self.name)

386

387 - def delete_host_template(self, name):

388 """ 389 Deletes a host template. 390 @param name: Name of the host template to delete. 391 @return: An ApiHostTemplate object. 392 """ 393 return host_templates.delete_host_template(self._get_resource_root(), name, self.name)

394

395 - def rolling_restart(self, slave_batch_size=None, 396 slave_fail_count_threshold=None, 397 sleep_seconds=None, 398 stale_configs_only=None, 399 unupgraded_only=None, 400 roles_to_include=None, 401 restart_service_names=None):

402 """ 403 Command to do a "best-effort" rolling restart of the given cluster, 404 i.e. it does plain restart of services that cannot be rolling restarted, 405 followed by first rolling restarting non-slaves and then rolling restarting 406 the slave roles of services that can be rolling restarted. The slave restarts 407 are done host-by-host. 408 @param slave_batch_size: Number of hosts with slave roles to restart at a time 409 Must be greater than 0. Default is 1. 410 @param slave_fail_count_threshold: The threshold for number of slave host batches that 411 are allowed to fail to restart before the entire command is considered failed. 412 Must be >= 0. Default is 0. 413 @param sleep_seconds: Number of seconds to sleep between restarts of slave host batches. 414 Must be >=0. Default is 0. 415 @param stale_configs_only: Restart roles with stale configs only. Default is false. 416 @param unupgraded_only: Restart roles that haven't been upgraded yet. Default is false. 417 @param roles_to_include: Role types to restart. Default is slave roles only. 418 @param restart_service_names: List of specific services to restart. 419 @return: Reference to the submitted command. 420 @since: API v4 421 """ 422 args = dict() 423 if slave_batch_size: 424 args['slaveBatchSize'] = slave_batch_size 425 if slave_fail_count_threshold: 426 args['slaveFailCountThreshold'] = slave_fail_count_threshold 427 if sleep_seconds: 428 args['sleepSeconds'] = sleep_seconds 429 if stale_configs_only: 430 args['staleConfigsOnly'] = stale_configs_only 431 if unupgraded_only: 432 args['unUpgradedOnly'] = unupgraded_only 433 if roles_to_include: 434 args['rolesToInclude'] = roles_to_include 435 if restart_service_names: 436 args['restartServiceNames'] = restart_service_names 437 438 return self._cmd('rollingRestart', data=args, api_version=4)

439

440 - def rolling_upgrade(self, upgrade_from_cdh_version, 441 upgrade_to_cdh_version, 442 upgrade_service_names, 443 slave_batch_size=None, 444 slave_fail_count_threshold=None, 445 sleep_seconds=None):

446 """ 447 Command to do a rolling upgrade of services in the given cluster 448 449 This command does not handle any services that don't support rolling 450 upgrades. The command will throw an error and not start if upgrade of 451 any such service is requested. 452 453 This command does not upgrade the full CDH Cluster. You should normally 454 use the upgradeCDH Command for upgrading the cluster. This is primarily 455 helpful if you need to need to recover from an upgrade failure or for 456 advanced users to script an alternative to the upgradeCdhCommand. 457 458 This command expects the binaries to be available on hosts and activated. 459 It does not change any binaries on the hosts. 460 461 @param upgrade_from_cdh_version: Current CDH Version of the services. 462 Example versions are: "5.1.0", "5.2.2" or "5.4.0" 463 @param upgrade_to_cdh_version: Target CDH Version for the services. 464 The CDH version should already be present and activated on the nodes. 465 Example versions are: "5.1.0", "5.2.2" or "5.4.0" 466 @param upgrade_service_names: List of specific services to be upgraded and restarted. 467 @param slave_batch_size: Number of hosts with slave roles to restart at a time 468 Must be greater than 0. Default is 1. 469 @param slave_fail_count_threshold: The threshold for number of slave host batches that 470 are allowed to fail to restart before the entire command is considered failed. 471 Must be >= 0. Default is 0. 472 @param sleep_seconds: Number of seconds to sleep between restarts of slave host batches. 473 Must be >=0. Default is 0. 474 475 @return: Reference to the submitted command. 476 @since: API v10 477 """ 478 args = dict() 479 args['upgradeFromCdhVersion'] = upgrade_from_cdh_version 480 args['upgradeToCdhVersion'] = upgrade_to_cdh_version 481 args['upgradeServiceNames'] = upgrade_service_names 482 483 if slave_batch_size: 484 args['slaveBatchSize'] = slave_batch_size 485 if slave_fail_count_threshold: 486 args['slaveFailCountThreshold'] = slave_fail_count_threshold 487 if sleep_seconds: 488 args['sleepSeconds'] = sleep_seconds 489 490 return self._cmd('rollingUpgrade', data=args, api_version=10)

491

492 - def auto_assign_roles(self):

493 """ 494 Automatically assign roles to hosts and create the roles for all the services in a cluster. 495 496 Assignments are done based on services in the cluster and hardware specifications. 497 Existing roles will be taken into account and their assignments will be not be modified. 498 @since: API v6 499 """ 500 self._put("autoAssignRoles", None, api_version=6)

501

502 - def auto_configure(self):

503 """ 504 Automatically configures roles and services in a cluster. 505 506 Overwrites some existing configurations. Might create new role config 507 groups. Only default role config groups must exist before calling this 508 endpoint. Other role config groups must not exist. If they do, an exception 509 will be thrown preventing any configuration. Ignores the Cloudera 510 Management Service even if colocated with roles of this cluster. To avoid 511 over-committing the heap on hosts, assign hosts to this cluster that are 512 not being used by the Cloudera Management Service. 513 @since: API v6 514 """ 515 self._put("autoConfigure", None, api_version=6)

516

517 - def first_run(self):

518 """ 519 Prepare and start services in a cluster. 520 Perform all the steps needed to prepare each service in a 521 cluster and start the services in order. 522 523 @return: Reference to the submitted command. 524 @since: API v7 525 """ 526 return self._cmd('firstRun', None, api_version=7)

527

528 - def upgrade_cdh(self, deploy_client_config=True, start_all_services=True, 529 cdh_parcel_version=None, cdh_package_version=None, 530 rolling_restart=False, slave_batch_size=None, sleep_seconds=None, 531 slave_fail_count_threshold=None):

532 """ 533 Perform CDH upgrade to the next major version. In v9+, also supports 534 minor CDH 5 upgrades (5.a.b to 5.x.y where x > a) and supports maintenance 535 release changes (a.b.x to a.b.y). 536 537 If using packages, CDH packages on all hosts of the cluster must be 538 manually upgraded before this command is issued. 539 540 The command will upgrade the services and their configuration to the 541 requested version. All running services will be stopped before proceeding, 542 unless rolling restart is requested and is available. 543 544 @param deploy_client_config: Whether to deploy client configurations 545 after the upgrade. Default is True. Has no effect in v9+; 546 client configurations are always deployed. 547 @param start_all_services: Whether to start all services after the upgrade. 548 Default is True. Has no effect in v9+; services are always 549 restarted. 550 @param cdh_parcel_version: If upgrading to parcels, the full version of an 551 already distributed parcel for the next CDH version. Default 552 is None. Example versions are: '5.0.0-1.cdh5.0.0.p0.11' or 553 '5.0.2-1.cdh5.0.2.p0.32'. 554 @param cdh_package_version: If upgrading to packages, the full version of an 555 already installed package for the next CDH version. Default 556 is None. Example versions are: '5.2.0' or '4.5.0'. Only available 557 since v9. 558 @param rolling_restart: If you'd like to do a rolling restart, set this to 559 True. Default is False. Only available since v9. 560 @param slave_batch_size: Controls the rolling restart slave batch size. 561 Only applicable when rolling_restart is True. 562 @param sleep_seconds: Controls how many seconds to sleep betweein rolling 563 restart batches. Only applicable when rolling_restart is True. 564 @param slave_fail_count_threshold: Controls how many slave restart failures 565 are tolerated in a rolling restart. Only applicable when 566 rolling_restart is True. 567 @return: Reference to the submitted command. 568 @since: API v6 for major upgrades only, v9 for maintenance and CDH 5 minor 569 releases. 570 """ 571 args = dict() 572 args['deployClientConfig'] = deploy_client_config 573 args['startAllServices'] = start_all_services 574 if cdh_parcel_version: 575 args['cdhParcelVersion'] = cdh_parcel_version 576 if cdh_package_version: 577 args['cdhPackageVersion'] = cdh_package_version 578 if rolling_restart: 579 args['rollingRestartArgs'] = { 580 'slaveBatchSize' : slave_batch_size, 581 'sleepSeconds' : sleep_seconds, 582 'slaveFailCountThreshold' : slave_fail_count_threshold 583 } 584 return self._cmd('upgradeCdh', data=args, api_version=6)

585

586 - def configure_for_kerberos(self, datanode_transceiver_port=None, 587 datanode_web_port=None):

588 """ 589 Command to configure the cluster to use Kerberos for authentication. 590 591 This command will configure all relevant services on a cluster for 592 Kerberos usage. This command will trigger a GenerateCredentials command 593 to create Kerberos keytabs for all roles in the cluster. 594 595 @param datanode_transceiver_port: The HDFS DataNode transceiver port to use. 596 This will be applied to all DataNode role configuration groups. If 597 not specified, this will default to 1004. 598 @param datanode_web_port: The HDFS DataNode web port to use. This will be 599 applied to all DataNode role configuration groups. If not specified, 600 this will default to 1006. 601 @return: Reference to the submitted command. 602 @since: API v11 603 """ 604 args = dict() 605 if datanode_transceiver_port: 606 args['datanodeTransceiverPort'] = datanode_transceiver_port 607 if datanode_web_port: 608 args['datanodeWebPort'] = datanode_web_port 609 return self._cmd('configureForKerberos', data=args, api_version=11)

610

611 - def export(self, export_auto_config=False):

612 """ 613 Export the cluster template for the given cluster. ccluster must have host 614 templates defined. It cluster does not have host templates defined it will 615 export host templates based on roles assignment. 616 617 @param export_auto_config: Also export auto configured configs 618 @return: Return cluster template 619 @since: API v12 620 """ 621 622 return self._get("export", ApiClusterTemplate, False, 623 params=dict(exportAutoConfig=export_auto_config), api_version=12)

624

625 - def pools_refresh(self):

626 """ 627 Refresh Dynamic Pools configurations for relevant services.. 628 629 @return: Reference to the submitted command. 630 @since: API v6 631 """ 632 return self._cmd('poolsRefresh', api_version=6)

633

634 - def list_dfs_services(self, view=None):

635 """ 636 List available DFS (distributed file system) services in a cluster. 637 @param view: View to materialize 638 @return: List of available distributed file system services in the cluster. 639 @since: API v12 640 """ 641 if view: 642 return self._get_resource_root().get("%s/%s?view=%s" % (self._path(), 'dfsServices', view)) 643 else: 644 return self._get_resource_root().get("%s/%s" % (self._path(), 'dfsServices'))

645

646 - def delete_credentials(self):

647 """ 648 Delete kerberos credentials for services belonging to this cluster. 649 650 @return: Reference to the submitted command. 651 @since: API v18 652 """ 653 return self._cmd('deleteCredentials', api_version=18)

Source Code for Module cm_api.endpoints.clusters