Fabric starting guide
After the BB14 conf, I am posting some snippets of fabric tasks. Good luck! Feel free to email me if you have questions.
# Fabric snippets post BB14 conf
# It should be obvious, but no warranty, expressed or otherwise is provided
# with this code. Use at your own risk. Always read and understand code before
# running it in your environment. Test test test.
# William Brown, Geraint Draheim and others: University of Adelaide
# william at adelaide.edu.au
####################################################################
#### WARNING: THIS CODE MAY NOT RUN DUE TO LACK OF IMPORT, DEPENDENCIES.
#### OR OTHER ENVIRONMENTAL CHANGES. THIS IS INTENTIONAL. YOU SHOULD
#### ADAPT SOME OF THESE TO YOUR OWN ENVIRONMENT AND UNDERSTAND THE CODE
####################################################################
## Decorators. These provide wrappers to functions to allow you to enforce state
# checks and warnings to users before they run. Here are some of the most useful
# we have developed.
def rnt_verbose(func):
"""
When added to a function, this will add implementation of a global VERBOSE
flag. The reason it's not a default, is because not every function is
converted to use it yet. Just run command:verbose=1
"""
@wraps(func)
def inner(*args, **kwargs):
if kwargs.pop("verbose", None) is not None:
global VERBOSE
VERBOSE = True
return func(*args, **kwargs)
return inner
## IMPORTANT NOTE: This decorator MUST be before @parallel
def rnt_imsure(warning=None):
"""
This is a useful decorator that enforces the user types a message into
the console before the task will run. This is invaluable for high risk
tasks, essentially forcing that the user MUST take responsibility for their
actions.
"""
def decorator(func):
@wraps(func)
def inner(*args, **kwargs):
# pylint: disable=global-statement
global IMSURE_WARNING
print("Forcing task to run in serial")
if kwargs.pop("imsure", None) is None and IMSURE_WARNING is False:
if warning is not None:
print(warning)
cont = getpass('If you are sure, type "I know what I am doing." #')
if cont == 'I know what I am doing.':
IMSURE_WARNING = True
print('continuing in 5 seconds ...')
time.sleep(5)
print("Starting ...")
else:
print('Exiting : No actions were taken')
sys.exit(1)
return func(*args, **kwargs)
inner.parallel = False
inner.serial = True
return inner
return decorator
def rnt_untested(func):
"""
This decorator wraps functions that we consider new and untested. It gives
a large, visual warning to the user that this is the case, and allows
5 seconds for them to ctrl+c before continuing.
"""
@wraps(func)
def inner(*args, **kwargs):
dragon = """
____________________________________
/ THIS IS AN UNTESTED TASK. THERE \\
\\ ARE DRAGONS IN THESE PARTS /
------------------------------------
\\ / \\ //\\
\\ |\\___/| / \\// \\\\
/0 0 \\__ / // | \\ \\
/ / \\/_/ // | \\ \\
@_^_@'/ \\/_ // | \\ \\
//_^_/ \\/_ // | \\ \\
( //) | \\/// | \\ \\
( / /) _|_ / ) // | \\ _\\
( // /) '/,_ _ _/ ( ; -. | _ _\\.-~ .-~~~^-.
(( / / )) ,-{ _ `-.|.-~-. .~ `.
(( // / )) '/\\ / ~-. _ .-~ .-~^-. \\
(( /// )) `. { } / \\ \\
(( / )) .----~-.\\ \\-' .~ \\ `. \\^-.
///.----..> \\ _ -~ `. ^-` ^-_
///-._ _ _ _ _ _ _}^ - - - - ~ ~-- ,.-~
/.-~
"""
# pylint: disable=global-statement
global DRAGON_WARNING
if not DRAGON_WARNING:
print(dragon)
if kwargs.pop("dragon", None) is None:
time.sleep(5)
print("RAWR: Your problem now!!!")
DRAGON_WARNING = True
return func(*args, **kwargs)
return inner
#################################################
# Atomic locking functions. Provides a full lock, and a read lock. This is so
# that multiple systems, users etc can access servers, but the servers allow
# one and only one action to be occuring.
ATOMIC_LOCK = "/tmp/fsm_atomic.lock"
ATOMIC_FLOCK = "/tmp/fsm_atomic.flock"
ATOMIC_LOCK_HOSTS = {}
LOCAL_HOSTNAME = socket.gethostname()
class AtomicException(Exception):
pass
@task
def lock():
"""
usage: lock
WARNING: DO NOT RUN THIS BY HAND UNLESS YOU KNOW WHAT YOU ARE DOING!!!
Will create the atomic FSM lock. This prevents any other atomic function
from being able to run.
"""
### I cannot stress enough, do not change this.
result = run("""
(
flock -n 9 || exit 1
touch {lock}
echo {hostname} > {lock}
) 9>{flock}
""".format(lock=ATOMIC_LOCK, flock=ATOMIC_FLOCK, hostname=LOCAL_HOSTNAME) )
if result.return_code == 0:
return True
return False
@task
def unlock():
"""
usage: unlock
WARNING: DO NOT RUN THIS BY HAND UNLESS YOU KNOW WHAT YOU ARE DOING!!!
Will remove the atomic FSM lock. This allows any other atomic function
from to run.
Only run this if you are sure that it needs to clean out a stale lock. The
fsm atomic wrapper is VERY GOOD at cleaning up after itself. Only a kill -9
to the fabric job will prevent it removing the atomic lock. Check what
you are doing! Look inside of /tmp/fsm_atomic.lock to see who holds the lock right now!
"""
### I cannot stress enough, do not change this.
result = run("""
rm {lock}
""".format(lock=ATOMIC_LOCK))
if result == 0:
return True
return False
def _lock_check():
# pylint: disable=global-statement
global ATOMIC_LOCK_HOSTS
atomic_lock = False
t_owner = False
if ATOMIC_LOCK_HOSTS.has_key(env.host_string):
atomic_lock = ATOMIC_LOCK_HOSTS[env.host_string]
t_owner = True
if not atomic_lock:
with hide('warnings', 'running'):
result = get(ATOMIC_LOCK, local_path="/tmp/{host}/{page}".format(
page="fsm_atomic.lock", host=env.host))
if len(result) != 0:
atomic_lock = True
return atomic_lock, t_owner
def noop(*args, **kwargs):
log_local('No-op for %s' % env.host_string, 'NOTICE')
def rnt_fsm_atomic_r(func):
"""
This decorator wraps functions that relate to the FSM and changing of state.
It triggers an atomic lock in the FSM to prevent other state changes occuring
Fsm atomic tasks can be nested, only the top level task will manage the lock.
If the lock is already taken, we will NOT allow the task to run.
"""
@wraps(func)
def inner(*args, **kwargs):
#If ATOMIC_LOCK_HOSTS then we own the lock, so we can use it.
# ELSE if we don't hold ATOMIC_LOCK_HOSTS we should check.
# Really, only the outer most wrapper should check ....
with settings(warn_only=True):
# pylint: disable=global-statement
global ATOMIC_LOCK_HOSTS
#We DO care about the thread owner. Consider an exclusive lock above
# a read lock. If we didn't check that we own that exclusive lock,
# we wouldn't be able to run.
(atomic_lock, t_owner) = _lock_check()
allow_run = False
if not atomic_lock or (atomic_lock and t_owner):
### We can run
allow_run = True
pass
elif atomic_lock and not t_owner:
### We can't run. The lock is held, and we don't own it.
log_local('ATOMIC LOCK EXISTS, CANNOT RUN %s' % env.host_string, 'NOTICE')
elif atomic_lock and t_owner:
#### THIS SHOULDN'T HAPPEN EVER
log_local('ATOMIC LOCK STATE IS INVALID PLEASE CHECK', 'CRITICAL')
raise AtomicException("CRITICAL: ATOIC LOCK STATE IS INVALID PLEASE CHECK, CANNOT RUN %s" % env.host_string)
elif not atomic_lock and not t_owner:
### This means there is no lock, and we don't own one. We can run.
pass
try:
if allow_run:
return func(*args, **kwargs)
else:
return noop(*args, **kwargs)
finally:
pass
return inner
def rnt_fsm_atomic_exc(func):
"""
This decorator wraps functions that relate to the FSM and changing of state.
It triggers an atomic lock in the FSM to prevent other state changes occuring
until the task is complete.
Fsm atomic tasks can be nested, only the top level task will manage the lock.
If the lock is already taken, we will NOT allow the task to run.
State is passed to nested calls that also need an atomic lock.
"""
@wraps(func)
def inner(*args, **kwargs):
with settings(warn_only=True):
# pylint: disable=global-statement
global ATOMIC_LOCK_HOSTS
(atomic_lock, t_owner) = _lock_check()
atomic_lock_owner = False
allow_run = False
if atomic_lock and t_owner:
#We have the lock, do nothing.
pass
allow_run = True
elif atomic_lock and not t_owner:
#Someone else has it, error.
log_local('ATOMIC LOCK EXISTS, CANNOT RUN %s' % env.host_string, 'IMPORTANT')
elif not atomic_lock and t_owner:
#Error, can't be in this state.
log_local('ATOMIC LOCK STATE IS INVALID PLEASE CHECK', 'CRITICAL')
raise AtomicException("CRITICAL: ATOMIC LOCK STATE IS INVALID PLEASE CHECK, CANNOT RUN %s" % env.host_string)
elif not atomic_lock and not t_owner:
# Create the lock.
if not lock():
log_local('LOCK TAKEN BY ANOTHER PROCESS', 'IMPORTANT')
raise AtomicException("CRITICAL: LOCK TAKEN BY ANOTHER PROCESS")
ATOMIC_LOCK_HOSTS[env.host_string] = True
atomic_lock_owner = True
allow_run = True
try:
if allow_run:
return func(*args, **kwargs)
else:
return noop(*args, **kwargs)
finally:
if atomic_lock_owner:
unlock()
ATOMIC_LOCK_HOSTS[env.host_string] = False
return inner
##################################################
# Basic service management.
#
## This is how you should start. Basic start, stop, and status commands.
@task
@parallel
def start():
"""
usage: start
Start the MapleTA database, tomcat and webserver
"""
sudo('service postgresql start')
sudo('service tomcat6 start')
sudo('service httpd start')
@task
@parallel
def stop():
"""
usage: stop
Stop the MapleTA webserver, tomcat and database
"""
sudo('service httpd stop')
sudo('service tomcat6 stop')
sudo('service postgresql stop')
@task
def restart():
"""
usage: restart
Restart the MapleTA database, tomcat and webserver
"""
stop()
start()
@task
def status():
"""
usage: status
Check the status of MapleTA
"""
sudo('service postgresql status')
sudo('service tomcat6 status')
sudo('service httpd status')
##################################
# Some blackboard tasks. These rely on some of the above decorators.
#
### These are well developed, and sometimes rely on code not provided here. This
# in very intentional so that you can read it and get ideas of HOW you should
# build code that works in your environment.
# Also shows the usage of decorators and how you should use them to protent tasks
###################################
# Helpers
###################################
def config_key(key):
if key.endswith('=') is False:
key += '='
return run("egrep '{key}' {bbconfig} | cut -f2 -d\= ".format(key=key, bbconfig=BB_CONFIG))
# return blackboard database instance
@task
@rnt_help
def get_db_instance():
"""
usage: get_db_instance
Display the servers current DB instance / SID
"""
x = config_key('bbconfig.database.server.instancename')
return x
@task
def get_db_credentials():
"""
usage: get_db_credentials
This will retrieve the DB username and password from the BB server, and
return them as a dict {hostname:X, sid:X, username:X, password:X}
"""
creds = {'hostname' : None,
'sid' : None,
'username' : None,
'password' : None}
with hide('everything'):
creds['hostname'] = config_key('bbconfig.database.server.fullhostname')
#TODO: Remove this sid appending line
creds['sid'] = config_key('bbconfig.database.server.instancename') + '.blackboard.inc'
creds['username'] = config_key('antargs.default.vi.db.name')
creds['password'] = config_key('antargs.default.vi.db.password')
return creds
@task
@parallel
@rnt_fsm_atomic_exc
def force_stop():
"""
usage: force_stop -> atomic
Stop blackboard services on hosts in PARALLEL. This WILL bring down all
hosts FAST. This does NOT gracefully remove from the pool. This DOES NOT
check the sis integration queue.
"""
log_blackboard("Stopping BB", level='NOTICE')
if test_processes(quit=False) is True:
sudo('/data/blackboard/bbctl stop')
time.sleep(30)
cleanup_processes()
test_processes()
log_blackboard("Stopped", level='SUCCESS')
@task
@serial
@rnt_fsm_atomic_exc
def force_restart():
"""
usage: restart -> atomic
Restart blackboard systems in SERIAL. This is a dumb rolling restart. This
DOES NOT remove from the pool and DOES NOT check the SIS queue
"""
log_blackboard("Trying to force restart blackboard", level='NOTICE')
force_stop()
time.sleep(60)
start()
log_blackboard("force restart complete", level='SUCCESS')
@task
@rnt_imsure()
def pushconfigupdates():
"""
usage: pushconfigupdates
Run the pushconfigupdates tool on a system.
Warning! Running PushConfigUpdates.sh deploys changes to bb-config.properties!
* This will result in an outage to the host(s) on which it is run!
* Be careful that bb-config.properties, and the xythos.properties configuration
files point to the correct database before you run this!
"""
sudo('/data/blackboard/tools/admin/PushConfigUpdates.sh')
@rnt_fsm_atomic_exc
def _compress_and_delete(path, fileglob, zipage=7, rmage=3660):
"""
This will compress logs up to 7 days, and delete older than 62 days.
The pattern is taken as:
/a/b*/c/d.*.txt
This is passed to find which will carry out the actions as sudo.
"""
with settings(warn_only=True):
sudo("find {path} -mtime +{zipage} -name '{fileglob}' -exec gzip '{{}}' \;".format(path=path, fileglob=fileglob, zipage=zipage))
sudo("find {path} -mtime +{rmage} -name '{fileglob}.gz' -exec rm '{{}}' \;".format(path=path, fileglob=fileglob, rmage=rmage))
@task
@rnt_help
@rnt_fsm_atomic_exc
def rotate_tomcat_logs():
"""
usage: rotate_tomcat_logs -> atomic
This will rotate the tomcat logs in /data/blackboard/logs/tomcat.
"""
log_blackboard(level="NOTICE")
with settings(warn_only=True):
for pattern in ['stdout-stderr-*.log', 'bb-access-log.*.txt',
'activemq.txt.*.txt', 'catalina-log.txt.*.txt', 'gc.*.txt',
'thread_dump*.txt', '*.hprof' ]:
_compress_and_delete("/data/blackboard/logs/tomcat/", pattern)