# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2020 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Authors:
# Santiago Dueñas <sduenas@bitergia.com>
# Stephan Barth <stephan.barth@gmail.com>
# Valerio Cosentino <valcos@bitergia.com>
# Jesus M. Gonzalez-Barahona <jgb@gsyc.es>
# Harshal Mittal <harshalmittal4@gmail.com>
#
import json
import logging
import requests
from grimoirelab_toolkit.datetime import datetime_to_utc, str_to_datetime
from grimoirelab_toolkit.uris import urijoin
from ...backend import (Backend,
BackendCommand,
BackendCommandArgumentParser)
from ...client import HttpClient
from ...utils import DEFAULT_DATETIME
CATEGORY_ISSUE = "issue"
MAX_ISSUES = 100 # Maximum number of issues per query
USER_FIELDS = ['assigned_to', 'author']
logger = logging.getLogger(__name__)
[docs]class Redmine(Backend):
"""Redmine backend.
This class allows to fetch the issues stored on a Redmine
server. Initialize this class passing the URL of this server.
Some servers require authentication to get access to some
data, if this is the case, pass the API token to `api_token`
parameter.
:param url: URL of the server
:param api_token: token needed to use the API
:param max_issues: maximum number of issues requested on the same query
:param tag: label used to mark the data
:param archive: archive to store/retrieve items
:param ssl_verify: enable/disable SSL verification
"""
version = '0.11.0'
CATEGORIES = [CATEGORY_ISSUE]
EXTRA_SEARCH_FIELDS = {
'project_name': ['project', 'name'],
'project_id': ['project', 'id']
}
def __init__(self, url, api_token=None, max_issues=MAX_ISSUES,
tag=None, archive=None, ssl_verify=True):
origin = url
super().__init__(origin, tag=tag, archive=archive, ssl_verify=ssl_verify)
self.url = url
self.api_token = api_token
self.max_issues = max_issues
self.client = None
self._users = {}
[docs] def fetch(self, category=CATEGORY_ISSUE, from_date=DEFAULT_DATETIME):
"""Fetch the issues from the server.
This method fetches the issues stored on the server that were
updated since the given date. Data about attachments, journals
and watchers (among others) are included within each issue.
:param category: the category of items to fetch
:param from_date: obtain issues updated since this date
:returns: a generator of issues
"""
if not from_date:
from_date = DEFAULT_DATETIME
from_date = datetime_to_utc(from_date)
kwargs = {'from_date': from_date}
items = super().fetch(category, **kwargs)
return items
[docs] def fetch_items(self, category, **kwargs):
"""Fetch the issues
:param category: the category of items to fetch
:param kwargs: backend arguments
:returns: a generator of items
"""
from_date = kwargs['from_date']
logger.info("Fetching issues of '%s' from %s",
self.url, str(from_date))
nissues = 0
for issue_id in self.__fetch_issues_ids(from_date):
issue = self.__fetch_and_parse_issue(issue_id)
for key in USER_FIELDS:
if key not in issue:
continue
user = self.__get_or_fetch_user(issue[key]['id'])
issue[key + '_data'] = user
for journal in issue['journals']:
if 'user' not in journal:
continue
user = self.__get_or_fetch_user(journal['user']['id'])
journal['user_data'] = user
yield issue
nissues += 1
logger.info("Fetch process completed: %s issues fetched", nissues)
[docs] @classmethod
def has_archiving(cls):
"""Returns whether it supports archiving items on the fetch process.
:returns: this backend supports items archive
"""
return True
[docs] @classmethod
def has_resuming(cls):
"""Returns whether it supports to resume the fetch process.
:returns: this backend supports items resuming
"""
return True
[docs] @staticmethod
def parse_issues(raw_json):
"""Parse a Redmine issues JSON stream.
The method parses a JSON stream and returns a list iterator.
Each item is a dictionary that contains the issue parsed data.
:param raw_json: JSON string to parse
:returns: a generator of parsed issues
"""
results = json.loads(raw_json)
issues = results['issues']
for issue in issues:
yield issue
[docs] @staticmethod
def parse_issue_data(raw_json):
"""Parse a Redmine issue JSON stream.
The method parses a JSON stream and returns a dictionary
with the parsed data for the given issue.
:param raw_json: JSON string to parse
:returns: a dictionary with the parsed issue data
"""
result = json.loads(raw_json)
return result['issue']
[docs] @staticmethod
def parse_user_data(raw_json):
"""Parse a Redmine user JSON stream.
The method parses a JSON stream and returns a dictionary
with the parsed data for the given user.
:param raw_json: JSON string to parse
:returns: a dictionary with the parsed user data
"""
result = json.loads(raw_json)
return result['user']
def _init_client(self, from_archive=False):
"""Init client"""
return RedmineClient(self.url, self.api_token, self.archive, from_archive, self.ssl_verify)
def __fetch_issues_ids(self, from_date):
offset = 0
issues = self.__fetch_and_parse_issues_page(from_date, offset,
self.max_issues)
while issues:
issue = issues.pop(0)
issue_id = issue['id']
yield issue_id
if not issues:
offset += self.max_issues
issues = self.__fetch_and_parse_issues_page(from_date, offset,
self.max_issues)
def __get_or_fetch_user(self, user_id):
if user_id in self._users:
return self._users[user_id]
logger.debug("User %s not found on client cache; fetching it", user_id)
try:
user = self.__fetch_and_parse_user(user_id)
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
logger.warning("User %s not found on the server; skipping it",
user_id)
user = {}
else:
raise e
self._users[user_id] = user
return user
def __fetch_and_parse_issues_page(self, from_date, offset, max_issues):
logger.debug("Fetching and parsing issues page from %s (offset: %s)",
str(from_date), str(offset))
raw_json = self.client.issues(from_date=from_date, offset=offset,
max_issues=max_issues)
issues = self.parse_issues(raw_json)
return [issue for issue in issues]
def __fetch_and_parse_issue(self, issue_id):
logger.debug("Fetching and parsing issue #%s", issue_id)
raw_issue = self.client.issue(issue_id)
return self.parse_issue_data(raw_issue)
def __fetch_and_parse_user(self, user_id):
logger.debug("Fetching and parsing user #%s", user_id)
raw_user = self.client.user(user_id)
return self.parse_user_data(raw_user)
[docs]class RedmineCommand(BackendCommand):
"""Class to run Redmine backend from the command line."""
BACKEND = Redmine
[docs] @classmethod
def setup_cmd_parser(cls):
"""Returns the Redmine argument parser."""
parser = BackendCommandArgumentParser(cls.BACKEND,
from_date=True,
token_auth=True,
archive=True,
ssl_verify=True)
# Redmine options
group = parser.parser.add_argument_group('Redmine arguments')
group.add_argument('--max-issues', dest='max_issues',
type=int, default=MAX_ISSUES,
help="Maximum number of issues requested on the same query")
# Required arguments
parser.parser.add_argument('url',
help="URL of the Redmine server")
return parser
[docs]class RedmineClient(HttpClient):
"""Redmine API client.
This class implements a client that retrieves issues from
a Redmine server. Remine servers provides a REST API that
returns its results in JSON format.
:param base_url: URL of the Phabricator server
:param api_token: token to get access to restricted data
stored in the server
:param archive: an archive to store/read fetched data
:param from_archive: it tells whether to write/read the archive
:param ssl_verify: enable/disable SSL verification
"""
URL = '%(base)s/%(resource)s'
RISSUES = 'issues'
RUSERS = 'users'
PINCLUDE = 'include'
PKEY = 'key'
PLIMIT = 'limit'
POFFSET = 'offset'
PSORT = 'sort'
PSTATUS_ID = 'status_id'
PUPDATED_ON = 'updated_on'
CJSON = '.json'
CATTACHMENTS = 'attachments'
CCHANGESETS = 'changesets'
CCHILDREN = 'children'
CJOURNALS = 'journals'
CRELATIONS = 'relations'
CWATCHERS = 'watchers'
def __init__(self, base_url, api_token=None, archive=None, from_archive=False, ssl_verify=True):
super().__init__(base_url.rstrip('/'), archive=archive, from_archive=from_archive, ssl_verify=ssl_verify)
self.api_token = api_token
[docs] def issues(self, from_date=DEFAULT_DATETIME,
offset=None, max_issues=MAX_ISSUES):
"""Get the information of a list of issues.
:param from_date: retrieve issues that where updated from that date;
dates are converted to UTC
:param offset: starting position for the search
:param max_issues: maximum number of issues to reteurn per query
"""
resource = self.RISSUES + self.CJSON
ts = datetime_to_utc(from_date)
ts = ts.strftime("%Y-%m-%dT%H:%M:%SZ")
# By default, Redmine returns open issues only.
# Parameter 'status_id' is set to get all the statuses.
params = {
self.PSTATUS_ID: '*',
self.PSORT: self.PUPDATED_ON,
self.PUPDATED_ON: '>=' + ts,
self.PLIMIT: max_issues
}
if offset is not None:
params[self.POFFSET] = offset
response = self._call(resource, params)
return response
[docs] def issue(self, issue_id):
"""Get the information of the given issue.
:param issue_id: issue identifier
"""
resource = urijoin(self.RISSUES, str(issue_id) + self.CJSON)
params = {
self.PINCLUDE: ','.join([self.CATTACHMENTS, self.CCHANGESETS,
self.CCHILDREN, self.CJOURNALS,
self.CRELATIONS, self.CWATCHERS])
}
response = self._call(resource, params)
return response
[docs] def user(self, user_id):
"""Get the information of the given user.
:param user_id: user identifier
"""
resource = urijoin(self.RUSERS, str(user_id) + self.CJSON)
params = {}
response = self._call(resource, params)
return response
[docs] @staticmethod
def sanitize_for_archive(url, headers, payload):
"""Sanitize payload of a HTTP request by removing the token information
before storing/retrieving archived items
:param: url: HTTP url request
:param: headers: HTTP headers request
:param: payload: HTTP payload request
:returns url, headers and the sanitized payload
"""
if RedmineClient.PKEY in payload:
payload.pop(RedmineClient.PKEY)
return url, headers, payload
def _call(self, resource, params):
"""Call to get a resource.
:param method: resource to get
:param params: dict with the HTTP parameters needed to get
the given resource
"""
url = self.URL % {'base': self.base_url, 'resource': resource}
if self.api_token:
params[self.PKEY] = self.api_token
logger.debug("Redmine client requests: %s params: %s",
resource, str(params))
r = self.fetch(url, payload=params)
return r.text