import json
import logging
import os
import tarfile
import StringIO
import tempfile
from time import strftime
from cgi import FieldStorage
from galaxy import util
from galaxy import web
from galaxy.exceptions import RequestParameterMissingException
from galaxy.exceptions import RequestParameterInvalidException
from galaxy.exceptions import InsufficientPermissionsException
from galaxy.exceptions import ActionInputError
from galaxy.exceptions import ObjectNotFound
from galaxy.exceptions import MalformedId
from galaxy.datatypes import checkers
from galaxy.model.orm import and_
from galaxy.web import _future_expose_api as expose_api
from galaxy.web import _future_expose_api_anonymous_and_sessionless as expose_api_anonymous_and_sessionless
from galaxy.web.base.controller import BaseAPIController
from galaxy.web.base.controller import HTTPBadRequest
from galaxy.web.framework.helpers import time_ago
from tool_shed.capsule import capsule_manager
from tool_shed.metadata import repository_metadata_manager
from tool_shed.repository_types import util as rt_util
from tool_shed.dependencies import attribute_handlers
from tool_shed.util import basic_util
from tool_shed.util import commit_util
from tool_shed.util import encoding_util
from tool_shed.util import hg_util
from tool_shed.util import repository_util
from tool_shed.util import repository_content_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_util
log = logging.getLogger( __name__ )
[docs]class RepositoriesController( BaseAPIController ):
"""RESTful controller for interactions with repositories in the Tool Shed."""
@web.expose_api
[docs] def add_repository_registry_entry( self, trans, payload, **kwd ):
"""
POST /api/repositories/add_repository_registry_entry
Adds appropriate entries to the repository registry for the repository defined by the received name and owner.
:param key: the user's API key
The following parameters are included in the payload.
:param tool_shed_url (required): the base URL of the Tool Shed containing the Repository
:param name (required): the name of the Repository
:param owner (required): the owner of the Repository
"""
response_dict = {}
if not trans.user_is_admin():
response_dict[ 'status' ] = 'error'
response_dict[ 'message' ] = "You are not authorized to add entries to this Tool Shed's repository registry."
return response_dict
tool_shed_url = payload.get( 'tool_shed_url', '' )
if not tool_shed_url:
raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." )
tool_shed_url = tool_shed_url.rstrip( '/' )
name = payload.get( 'name', '' )
if not name:
raise HTTPBadRequest( detail="Missing required parameter 'name'." )
owner = payload.get( 'owner', '' )
if not owner:
raise HTTPBadRequest( detail="Missing required parameter 'owner'." )
repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
if repository is None:
error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) )
log.debug( error_message )
response_dict[ 'status' ] = 'error'
response_dict[ 'message' ] = error_message
return response_dict
# Update the repository registry.
trans.app.repository_registry.add_entry( repository )
response_dict[ 'status' ] = 'ok'
response_dict[ 'message' ] = 'Entries for repository %s owned by %s have been added to the Tool Shed repository registry.' \
% ( name, owner )
return response_dict
@web.expose_api_anonymous
[docs] def get_ordered_installable_revisions( self, trans, name, owner, **kwd ):
"""
GET /api/repositories/get_ordered_installable_revisions
:param name: the name of the Repository
:param owner: the owner of the Repository
Returns the ordered list of changeset revision hash strings that are associated with installable revisions.
As in the changelog, the list is ordered oldest to newest.
"""
# Example URL: http://localhost:9009/api/repositories/get_installable_revisions?name=add_column&owner=test
if name and owner:
# Get the repository information.
repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
if repository is None:
error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: "
error_message += "cannot locate repository %s owned by %s." % ( str( name ), str( owner ) )
log.debug( error_message )
return []
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
ordered_installable_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True )
return ordered_installable_revisions
else:
error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: "
error_message += "invalid name %s or owner %s received." % ( str( name ), str( owner ) )
log.debug( error_message )
return []
@web.expose_api_anonymous
[docs] def get_repository_revision_install_info( self, trans, name, owner, changeset_revision, **kwd ):
"""
GET /api/repositories/get_repository_revision_install_info
:param name: the name of the Repository
:param owner: the owner of the Repository
:param changeset_revision: the changeset_revision of the RepositoryMetadata object associated with the Repository
Returns a list of the following dictionaries::
- a dictionary defining the Repository. For example:
{
"deleted": false,
"deprecated": false,
"description": "add_column hello",
"id": "f9cad7b01a472135",
"long_description": "add_column hello",
"name": "add_column",
"owner": "test",
"private": false,
"times_downloaded": 6,
"url": "/api/repositories/f9cad7b01a472135",
"user_id": "f9cad7b01a472135"
}
- a dictionary defining the Repository revision (RepositoryMetadata). For example:
{
"changeset_revision": "3a08cc21466f",
"downloadable": true,
"has_repository_dependencies": false,
"has_repository_dependencies_only_if_compiling_contained_td": false,
"id": "f9cad7b01a472135",
"includes_datatypes": false,
"includes_tool_dependencies": false,
"includes_tools": true,
"includes_tools_for_display_in_tool_panel": true,
"includes_workflows": false,
"malicious": false,
"repository_id": "f9cad7b01a472135",
"url": "/api/repository_revisions/f9cad7b01a472135"
}
- a dictionary including the additional information required to install the repository. For example:
{
"add_column": [
"add_column hello",
"http://test@localhost:9009/repos/test/add_column",
"3a08cc21466f",
"1",
"test",
{},
{}
]
}
"""
# Example URL:
# http://<xyz>/api/repositories/get_repository_revision_install_info?name=<n>&owner=<o>&changeset_revision=<cr>
if name and owner and changeset_revision:
# Get the repository information.
repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
if repository is None:
log.debug( 'Cannot locate repository %s owned by %s' % ( str( name ), str( owner ) ) )
return {}, {}, {}
encoded_repository_id = trans.security.encode_id( repository.id )
repository_dict = repository.to_dict( view='element',
value_mapper=self.__get_value_mapper( trans ) )
repository_dict[ 'url' ] = web.url_for( controller='repositories',
action='show',
id=encoded_repository_id )
# Get the repository_metadata information.
repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app,
encoded_repository_id,
changeset_revision )
if repository_metadata is None:
# The changeset_revision column in the repository_metadata table has been updated with a new
# value value, so find the changeset_revision to which we need to update.
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision )
repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app,
encoded_repository_id,
new_changeset_revision )
changeset_revision = new_changeset_revision
if repository_metadata is not None:
encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id )
repository_metadata_dict = repository_metadata.to_dict( view='collection',
value_mapper=self.__get_value_mapper( trans ) )
repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
action='show',
id=encoded_repository_metadata_id )
# Get the repo_info_dict for installing the repository.
repo_info_dict, \
includes_tools, \
includes_tool_dependencies, \
includes_tools_for_display_in_tool_panel, \
has_repository_dependencies, \
has_repository_dependencies_only_if_compiling_contained_td = \
repository_util.get_repo_info_dict( trans.app,
trans.user,
encoded_repository_id,
changeset_revision )
return repository_dict, repository_metadata_dict, repo_info_dict
else:
log.debug( "Unable to locate repository_metadata record for repository id %s and changeset_revision %s" % \
( str( repository.id ), str( changeset_revision ) ) )
return repository_dict, {}, {}
else:
debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: "
debug_msg += "Invalid name %s or owner %s or changeset_revision %s received." % \
( str( name ), str( owner ), str( changeset_revision ) )
log.debug( debug_msg )
return {}, {}, {}
def __get_value_mapper( self, trans ):
value_mapper = { 'id' : trans.security.encode_id,
'repository_id' : trans.security.encode_id,
'user_id' : trans.security.encode_id }
return value_mapper
@web.expose_api
[docs] def import_capsule( self, trans, payload, **kwd ):
"""
POST /api/repositories/new/import_capsule
Import a repository capsule into the Tool Shed.
:param key: the user's API key
The following parameters are included in the payload.
:param tool_shed_url (required): the base URL of the Tool Shed into which the capsule should be imported.
:param capsule_file_name (required): the name of the capsule file.
"""
# Get the information about the capsule to be imported from the payload.
tool_shed_url = payload.get( 'tool_shed_url', '' )
if not tool_shed_url:
raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." )
capsule_file_name = payload.get( 'capsule_file_name', '' )
if not capsule_file_name:
raise HTTPBadRequest( detail="Missing required parameter 'capsule_file_name'." )
capsule_file_path = os.path.abspath( capsule_file_name )
capsule_dict = dict( error_message='',
encoded_file_path=None,
status='ok',
tar_archive=None,
uploaded_file=None,
capsule_file_name=None )
if os.path.getsize( os.path.abspath( capsule_file_name ) ) == 0:
log.debug( 'Your capsule file %s is empty.' % str( capsule_file_name ) )
return {}
try:
# Open for reading with transparent compression.
tar_archive = tarfile.open( capsule_file_path, 'r:*' )
except tarfile.ReadError, e:
log.debug( 'Error opening capsule file %s: %s' % ( str( capsule_file_name ), str( e ) ) )
return {}
irm = capsule_manager.ImportRepositoryManager( trans.app,
trans.request.host,
trans.user,
trans.user_is_admin() )
capsule_dict[ 'tar_archive' ] = tar_archive
capsule_dict[ 'capsule_file_name' ] = capsule_file_name
capsule_dict = irm.extract_capsule_files( **capsule_dict )
capsule_dict = irm.validate_capsule( **capsule_dict )
status = capsule_dict.get( 'status', 'error' )
if status == 'error':
log.debug( 'The capsule contents are invalid and cannot be imported:<br/>%s' % \
str( capsule_dict.get( 'error_message', '' ) ) )
return {}
encoded_file_path = capsule_dict.get( 'encoded_file_path', None )
if encoded_file_path is None:
log.debug( 'The capsule_dict %s is missing the required encoded_file_path entry.' % str( capsule_dict ) )
return {}
file_path = encoding_util.tool_shed_decode( encoded_file_path )
export_info_file_path = os.path.join( file_path, 'export_info.xml' )
export_info_dict = irm.get_export_info_dict( export_info_file_path )
manifest_file_path = os.path.join( file_path, 'manifest.xml' )
# The manifest.xml file has already been validated, so no error_message should be returned here.
repository_info_dicts, error_message = irm.get_repository_info_from_manifest( manifest_file_path )
# Determine the status for each exported repository archive contained within the capsule.
repository_status_info_dicts = irm.get_repository_status_from_tool_shed( repository_info_dicts )
# Generate a list of repository name / import results message tuples for display after the capsule is imported.
import_results_tups = []
# Only create repositories that do not yet exist and that the current user is authorized to create. The
# status will be None for repositories that fall into the intersection of these 2 categories.
for repository_status_info_dict in repository_status_info_dicts:
# Add the capsule_file_name and encoded_file_path to the repository_status_info_dict.
repository_status_info_dict[ 'capsule_file_name' ] = capsule_file_name
repository_status_info_dict[ 'encoded_file_path' ] = encoded_file_path
import_results_tups = irm.create_repository_and_import_archive( repository_status_info_dict,
import_results_tups )
irm.check_status_and_reset_downloadable( import_results_tups )
basic_util.remove_dir( file_path )
# NOTE: the order of installation is defined in import_results_tups, but order will be lost
# when transferred to return_dict.
return_dict = {}
for import_results_tup in import_results_tups:
ok, name_owner, message = import_results_tup
name, owner = name_owner
key = 'Archive of repository "%s" owned by "%s"' % ( str( name ), str( owner ) )
val = message.replace( '<b>', '"' ).replace( '</b>', '"' )
return_dict[ key ] = val
return return_dict
@web.expose_api_anonymous
[docs] def index( self, trans, deleted=False, owner=None, name=None, **kwd ):
"""
GET /api/repositories
:param deleted: True/False, displays repositories that are or are not set to deleted.
:param owner: the owner's public username.
:param name: the repository name.
Displays a collection (list) of repositories.
"""
# Example URL: http://localhost:9009/api/repositories
repository_dicts = []
deleted = util.asbool( deleted )
clause_list = [ and_( trans.app.model.Repository.table.c.deprecated == False,
trans.app.model.Repository.table.c.deleted == deleted ) ]
if owner is not None:
clause_list.append( and_( trans.app.model.User.table.c.username == owner,
trans.app.model.Repository.table.c.user_id == trans.app.model.User.table.c.id ) )
if name is not None:
clause_list.append( trans.app.model.Repository.table.c.name == name )
for repository in trans.sa_session.query( trans.app.model.Repository ) \
.filter( *clause_list ) \
.order_by( trans.app.model.Repository.table.c.name ):
repository_dict = repository.to_dict( view='collection',
value_mapper=self.__get_value_mapper( trans ) )
repository_dict[ 'category_ids' ] = \
[ trans.security.encode_id( x.category.id ) for x in repository.categories ]
repository_dicts.append( repository_dict )
return repository_dicts
@web.expose_api
[docs] def remove_repository_registry_entry( self, trans, payload, **kwd ):
"""
POST /api/repositories/remove_repository_registry_entry
Removes appropriate entries from the repository registry for the repository defined by the received name and owner.
:param key: the user's API key
The following parameters are included in the payload.
:param tool_shed_url (required): the base URL of the Tool Shed containing the Repository
:param name (required): the name of the Repository
:param owner (required): the owner of the Repository
"""
response_dict = {}
if not trans.user_is_admin():
response_dict[ 'status' ] = 'error'
response_dict[ 'message' ] = "You are not authorized to remove entries from this Tool Shed's repository registry."
return response_dict
tool_shed_url = payload.get( 'tool_shed_url', '' )
if not tool_shed_url:
raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." )
tool_shed_url = tool_shed_url.rstrip( '/' )
name = payload.get( 'name', '' )
if not name:
raise HTTPBadRequest( detail="Missing required parameter 'name'." )
owner = payload.get( 'owner', '' )
if not owner:
raise HTTPBadRequest( detail="Missing required parameter 'owner'." )
repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
if repository is None:
error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) )
log.debug( error_message )
response_dict[ 'status' ] = 'error'
response_dict[ 'message' ] = error_message
return response_dict
# Update the repository registry.
trans.app.repository_registry.remove_entry( repository )
response_dict[ 'status' ] = 'ok'
response_dict[ 'message' ] = 'Entries for repository %s owned by %s have been removed from the Tool Shed repository registry.' \
% ( name, owner )
return response_dict
@web.expose_api
@web.expose_api
@web.expose_api
@expose_api_anonymous_and_sessionless
[docs] def show( self, trans, id, **kwd ):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
:param id: the encoded id of the Repository object
:type id: encoded str
:returns: detailed repository information
:rtype: dict
:raises: ObjectNotFound, MalformedId
"""
try:
trans.security.decode_id( id )
except Exception:
raise MalformedId( 'The given id is invalid.' )
repository = suc.get_repository_in_tool_shed( trans.app, id )
if repository is None:
raise ObjectNotFound( 'Unable to locate repository for the given id.' )
repository_dict = repository.to_dict( view='element',
value_mapper=self.__get_value_mapper( trans ) )
# TODO the following property would be better suited in the to_dict method
repository_dict[ 'category_ids' ] = \
[ trans.security.encode_id( x.category.id ) for x in repository.categories ]
return repository_dict
@expose_api
[docs] def update( self, trans, id, **kwd ):
"""
PATCH /api/repositories/{encoded_repository_id}
Updates information about a repository in the Tool Shed.
:param id: the encoded id of the Repository object
:param payload: dictionary structure containing::
'name': repo's name (optional)
'synopsis': repo's synopsis (optional)
'description': repo's description (optional)
'remote_repository_url': repo's remote repo (optional)
'homepage_url': repo's homepage url (optional)
'category_ids': list of existing encoded TS category ids
the updated repo should be associated with (optional)
:type payload: dict
:returns: detailed repository information
:rtype: dict
:raises: RequestParameterInvalidException, InsufficientPermissionsException
"""
payload = kwd.get( 'payload', None )
if not payload:
raise RequestParameterMissingException( "You did not specify any payload." )
name = payload.get( 'name', None )
synopsis = payload.get( 'synopsis', None )
description = payload.get( 'description', None )
remote_repository_url = payload.get( 'remote_repository_url', None )
homepage_url = payload.get( 'homepage_url', None )
category_ids = payload.get( 'category_ids', None )
if category_ids is not None:
# We need to know if it was actually passed, and listify turns None into []
category_ids = util.listify( category_ids )
update_kwds = dict(
name=name,
description=synopsis,
long_description=description,
remote_repository_url=remote_repository_url,
homepage_url=homepage_url,
category_ids=category_ids,
)
repo, message = repository_util.update_repository( app=trans.app, trans=trans, id=id, **update_kwds )
if repo is None:
if "You are not the owner" in message:
raise InsufficientPermissionsException( message )
else:
raise ActionInputError( message )
repository_dict = repo.to_dict( view='element', value_mapper=self.__get_value_mapper( trans ) )
repository_dict[ 'category_ids' ] = \
[ trans.security.encode_id( x.category.id ) for x in repo.categories ]
return repository_dict
@expose_api
[docs] def create( self, trans, **kwd ):
"""
create( self, trans, payload, **kwd )
* POST /api/repositories:
Creates a new repository.
Only ``name`` and ``synopsis`` parameters are required.
:param payload: dictionary structure containing::
'name': new repo's name (required)
'synopsis': new repo's synopsis (required)
'description': new repo's description (optional)
'remote_repository_url': new repo's remote repo (optional)
'homepage_url': new repo's homepage url (optional)
'category_ids[]': list of existing encoded TS category ids
the new repo should be associated with (optional)
'type': new repo's type, defaults to ``unrestricted`` (optional)
:type payload: dict
:returns: detailed repository information
:rtype: dict
:raises: RequestParameterMissingException, RequestParameterInvalidException
"""
payload = kwd.get( 'payload', None )
if not payload:
raise RequestParameterMissingException( "You did not specify any payload." )
name = payload.get( 'name', None )
if not name:
raise RequestParameterMissingException( "Missing required parameter 'name'." )
synopsis = payload.get( 'synopsis', None )
if not synopsis:
raise RequestParameterMissingException( "Missing required parameter 'synopsis'." )
description = payload.get( 'description', '' )
remote_repository_url = payload.get( 'remote_repository_url', '' )
homepage_url = payload.get( 'homepage_url', '' )
category_ids = util.listify( payload.get( 'category_ids[]', '' ) )
selected_categories = [ trans.security.decode_id( id ) for id in category_ids ]
repo_type = payload.get( 'type', rt_util.UNRESTRICTED )
if repo_type not in rt_util.types:
raise RequestParameterInvalidException( 'This repository type is not valid' )
invalid_message = repository_util.validate_repository_name( trans.app, name, trans.user )
if invalid_message:
raise RequestParameterInvalidException( invalid_message )
repo, message = repository_util.create_repository( app=trans.app,
name=name,
type=repo_type,
description=synopsis,
long_description=description,
user_id = trans.user.id,
category_ids=category_ids,
remote_repository_url=remote_repository_url,
homepage_url=homepage_url )
repository_dict = repo.to_dict( view='element', value_mapper=self.__get_value_mapper( trans ) )
repository_dict[ 'category_ids' ] = \
[ trans.security.encode_id( x.category.id ) for x in repo.categories ]
return repository_dict
@web.expose_api
[docs] def create_changeset_revision( self, trans, id, payload, **kwd ):
"""
POST /api/repositories/{encoded_repository_id}/changeset_revision
Create a new tool shed repository commit - leaving PUT on parent
resource open for updating meta-attributes of the repository (and
Galaxy doesn't allow PUT multipart data anyway
https://trello.com/c/CQwmCeG6).
:param id: the encoded id of the Repository object
The following parameters may be included in the payload.
:param commit_message: hg commit message for update.
"""
# Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
rdah = attribute_handlers.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False )
tdah = attribute_handlers.ToolDependencyAttributeHandler( trans.app, unpopulate=False )
repository = suc.get_repository_in_tool_shed( trans.app, id )
repo_dir = repository.repo_path( trans.app )
repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False )
upload_point = commit_util.get_upload_point( repository, **kwd )
tip = repository.tip( trans.app )
file_data = payload.get('file')
# Code stolen from gx's upload_common.py
if isinstance( file_data, FieldStorage ):
assert not isinstance( file_data.file, StringIO.StringIO )
assert file_data.file.name != '<fdopen>'
local_filename = util.mkstemp_ln( file_data.file.name, 'upload_file_data_' )
file_data.file.close()
file_data = dict( filename=file_data.filename,
local_filename=local_filename )
elif type( file_data ) == dict and 'local_filename' not in file_data:
raise Exception( 'Uploaded file was encoded in a way not understood.' )
commit_message = kwd.get( 'commit_message', 'Uploaded' )
uploaded_file = open(file_data['local_filename'], 'rb')
uploaded_file_name = file_data['local_filename']
isgzip = False
isbz2 = False
isgzip = checkers.is_gzip( uploaded_file_name )
if not isgzip:
isbz2 = checkers.is_bz2( uploaded_file_name )
if ( isgzip or isbz2 ):
# Open for reading with transparent compression.
tar = tarfile.open( uploaded_file_name, 'r:*' )
else:
tar = tarfile.open( uploaded_file_name )
new_repo_alert = False
remove_repo_files_not_in_tar = True
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
repository_content_util.upload_tar(
trans,
rdah,
tdah,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert
)
if ok:
# Update the repository files for browsing.
hg_util.update_repository( repo )
# Get the new repository tip.
if tip == repository.tip( trans.app ):
trans.response.status = 400
message = 'No changes to repository.'
ok = False
else:
rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app,
user=trans.user,
repository=repository )
status, error_message = \
rmm.set_repository_metadata_due_to_new_tip( trans.request.host,
content_alert_str=content_alert_str,
**kwd )
if error_message:
ok = False
trans.response.status = 500
message = error_message
else:
trans.response.status = 500
if not ok:
return {
"err_msg": message,
"content_alert": content_alert_str
}
else:
return {
"message": message,
"content_alert": content_alert_str
}