def parse_nodeinfo_document(doc, host):
    result = deepcopy(defaults)
    nodeinfo_version = doc.get('version', '1.0')
    result['host'] = host
    result['name'] = doc.get('metadata', {}).get('nodeName', host)
    result['version'] = doc.get('software', {}).get('version', '')
    result['platform'] = doc.get('software', {}).get('name', 'unknown').lower()
    if nodeinfo_version in ('1.0', '1.1'):
        inbound = doc.get('protocols', {}).get('inbound', [])
        outbound = doc.get('protocols', {}).get('outbound', [])
        protocols = sorted(list(set(inbound + outbound)))
        result['protocols'] = protocols
    else:
        result['protocols'] = doc.get('protocols', [])
    inbound = doc.get('services', {}).get('inbound', [])
    outbound = doc.get('services', {}).get('outbound', [])
    services = sorted(list(set(inbound + outbound)))
    result['services'] = services
    result['open_signups'] = doc.get('openRegistrations', False)
    result['activity']['users']['total'] = int_or_none(doc.get('usage', {}).get('users', {}).get('total'))
    result['activity']['users']['half_year'] = int_or_none(doc.get('usage', {}).get('users', {}).get('activeHalfyear'))
    monthly = int_or_none(doc.get('usage', {}).get('users', {}).get('activeMonth'))
    result['activity']['users']['monthly'] = monthly
    if monthly:
        result['activity']['users']['weekly'] = int(monthly * MONTHLY_USERS_WEEKLY_MULTIPLIER)
    result['activity']['local_posts'] = int_or_none(doc.get('usage', {}).get('localPosts'))
    result['activity']['local_comments'] = int_or_none(doc.get('usage', {}).get('localComments'))
    result['features'] = doc.get('metadata', {})
    admin_handle = doc.get('metadata', {}).get('adminAccount', None)
    if admin_handle:
        result['organization']['account'] = f"{admin_handle}@{host}"
    return result def parse_nodeinfo2_document(doc, host):
    result = deepcopy(defaults)
    result['platform'] = doc.get('server', {}).get('software', 'unknown').lower()
    result['version'] = doc.get('server', {}).get('version', '') or ''
    # Ensure baseUrl is reported as the host we called
    base_url = doc.get('server', {}).get('baseUrl', '').rstrip('/')
    cleaned_base_url = re.sub(r'https?://', '', base_url).split(':')[0]
    if cleaned_base_url.startswith(host):
        result['host'] = cleaned_base_url
    else:
        raise ValueError('NodeInfo2 baseUrl is outside called host.')
    result['name'] = doc.get('server', {}).get('name', '') or ''
    if not result['name']:
        result['name'] = result['host']
    result['open_signups'] = doc.get('openRegistrations', False)
    inbound = doc.get('services', {}).get('inbound', [])
    outbound = doc.get('services', {}).get('outbound', [])
    services = sorted(list(set(inbound + outbound)))
    result['services'] = services
    result['protocols'] = sorted(doc.get('protocols', []))
    result['relay'] = doc.get('relay', '') or ''
    result['organization']['contact'] = doc.get('organization', {}).get('contact', '') or ''
    result['organization']['name'] = doc.get('organization', {}).get('name', '') or ''
    result['organization']['account'] = doc.get('organization', {}).get('account', '') or ''
    result['features'] = doc.get('otherFeatures', {})
    result['activity']['users']['total'] = int_or_none(doc.get('usage', {}).get('users', {}).get('total'))
    result['activity']['users']['half_year'] = int_or_none(doc.get('usage', {}).get('users', {}).get('activeHalfyear'))
    result['activity']['users']['monthly'] = int_or_none(doc.get('usage', {}).get('users', {}).get('activeMonth'))
    result['activity']['users']['weekly'] = int_or_none(doc.get('usage', {}).get('users', {}).get('activeWeek'))
    result['activity']['local_posts'] = int_or_none(doc.get('usage', {}).get('localPosts'))
    result['activity']['local_comments'] = int_or_none(doc.get('usage', {}).get('localComments'))
    return result def parse_statisticsjson_document(doc, host):
    result = deepcopy(defaults)
    result['host'] = host
    result['name'] = doc.get('name', '')
    result['platform'] = doc.get('network', 'unknown').lower()
    result['version'] = doc.get('version', '')
    result['open_signups'] = doc.get('registrations_open', False)
    result['services'] = sorted(doc.get('services', []))
    result['protocols'] = ['diaspora']  # Reasonable default
    result['activity']['users']['total'] = int_or_none(doc.get('total_users'))
    result['activity']['users']['half_year'] = int_or_none(doc.get('active_users_halfyear'))
    monthly = int_or_none(doc.get('active_users_monthly'))
    result['activity']['users']['monthly'] = monthly
    if monthly:
        result['activity']['users']['weekly'] = int(monthly * MONTHLY_USERS_WEEKLY_MULTIPLIER)
    result['activity']['local_posts'] = int_or_none(doc.get('local_posts'))
    result['activity']['local_comments'] = int_or_none(doc.get('local_comments'))
    return result def int_or_none(value):
    if isinstance(value, dict):
        return None
    if value is None or value == '':
        return None
    try:
        result = int(value)
        if result < 0:
            return None
        return result
    except ValueError:
        return None def parse_mastodon_document(doc, host):
    # Check first this is not actually Pleroma or Misskey
    if doc.get('version', '').find('Pleroma') > -1 or doc.get('version', '').find('Pixelfed') > -1 or \
            doc.get('version', '').find('Kibou') > -1 or doc.get('version', '').find('Kroeg') > -1:
        # Use NodeInfo instead, otherwise this is logged as Mastodon
        from federation.hostmeta.fetchers import fetch_nodeinfo_document
        return fetch_nodeinfo_document(host)
    elif doc.get('version', '').find('misskey') > -1:
        # Use Misskey instead, otherwise this is logged as Mastodon
        from federation.hostmeta.fetchers import fetch_misskey_document
        return fetch_misskey_document(host, mastodon_document=doc)

    result = deepcopy(defaults)
    result['host'] = host
    result['name'] = doc.get('title', host)
    result['platform'] = 'mastodon'
    result['version'] = doc.get('version', '')

    # Awkward parsing of signups from about page
    # TODO remove if fixed, issue logged: https://github.com/tootsuite/mastodon/issues/9350
    about_doc, _status_code, _error = fetch_document(host=host, path='/about')
    if about_doc:
        result['open_signups'] = about_doc.find("<div class='closed-registrations-message'>") == -1

    version = re.sub(r'[^0-9.]', '', doc.get('version', ''))
    version = [int(part) for part in version.split('.')]
    if version >= [3, 0, 0]:
        result['protocols'] = ['activitypub']
    elif version >= [1, 6, 0]:
        result['protocols'] = ['ostatus', 'activitypub']
    else:
        result['protocols'] = ['ostatus']
    result['relay'] = False

    result['activity']['users']['total'] = int_or_none(doc.get('stats', {}).get('user_count'))
    result['activity']['local_posts'] = int_or_none(doc.get('stats', {}).get('status_count'))

    if "contact_account" in doc and doc.get('contact_account') is not None:
        contact_account = doc.get('contact_account', {})
    else:
        contact_account = {}
    result['organization']['account'] = contact_account.get('url', '')
    result['organization']['contact'] = doc.get('email', '')
    result['organization']['name'] = contact_account.get('display_name', '')

    activity_doc, _status_code, _error = fetch_document(host=host, path='/api/v1/instance/activity')
    if activity_doc:
        try:
            activity_doc = json.loads(activity_doc)
        except json.JSONDecodeError:
            return result
        else:
            try:
                logins = activity_doc[1].get('logins')
            except KeyError:
                logins = activity_doc[0].get('logins')
            weekly_count = int_or_none(logins)
            if weekly_count and result['activity']['users']['total']:
                result['activity']['users']['weekly'] = weekly_count
                # Ensure multiplied counts from weekly count don't go over total user count
                result['activity']['users']['half_year'] = min(
                    int(weekly_count * WEEKLY_USERS_HALFYEAR_MULTIPLIER),
                    result['activity']['users']['total'],
                )
                result['activity']['users']['monthly'] = min(
                    int(weekly_count * WEEKLY_USERS_MONTHLY_MULTIPLIER),
                    result['activity']['users']['total'],
                )

    return result def parse_matrix_document(doc: Dict, host: str) -> Dict:
    result = deepcopy(defaults)
    result['host'] = host
    result['name'] = host
    result['protocols'] = ['matrix']
    result['platform'] = f'matrix|{doc["server"]["name"].lower()}'
    result['version'] = doc["server"]["version"]

    # Get signups status by posting to register endpoint and analyzing the status code coming back
    status_code, _error = send_document(
        f'https://{host}/_matrix/client/r0/register',
        data=json.dumps({'auth': {}}),
    )
    if status_code == 401:
        result['open_signups'] = True
    elif status_code == 403:
        result['open_signups'] = False

    return result NODEINFO2_10_DOC = """
{
  "version": "1.0",
  "server": {
    "baseUrl": "https://example.com/",
    "name": "Example server",
    "software": "example",
    "version": "0.5.0"
  },
  "organization": {
    "name": "Example organization",
    "contact": "foobar@example.com",
    "account": "https://example.com/u/admin"
  },
  "protocols": ["diaspora", "zot"],
  "relay": "tags",
  "services": {
    "inbound": ["gnusocial"],
    "outbound": ["facebook", "twitter"]
  },
  "openRegistrations": true,
  "usage": {
    "users": {
      "total": 123,
      "activeHalfyear": 42,
      "activeMonth": 23,
      "activeWeek": 10
    },
    "localPosts": 500,
    "localComments": 1000
  }
}
""" NODEINFO_10_DOC = '{"version":"1.0","software":{"name":"diaspora","version":"0.7.4.0-pd0313756"},"protocols":' \
                  '{"inbound":["diaspora"],"outbound":["diaspora"]},"services":{"inbound":[],"outbound":["twi' \
                  'tter","tumblr"]},"openRegistrations":true,"usage":{"users":{"total":348,"activeHalfyear":1' \
                  '23,"activeMonth":62},"localPosts":8522,"localComments":17671},"metadata":{"nodeName":"I Li' \
                  'ke Toast","xmppChat":false,"camo":{"markdown":false,"opengraph":false,"remotePods":false},' \
                  '"adminAccount":"podmin"}}' NODEINFO_20_DOC = '{"version":"2.0","software":{"name":"diaspora","version":"0.7.4.0-pd0313756"},"protocols":' \
                  '["diaspora"],"services":{"inbound":[],"outbound":["twitter","tumblr"]},"openRegistrations"' \
                  ':true,"usage":{"users":{"total":348,"activeHalfyear":123,"activeMonth":62},"localPosts":85' \
                  '22,"localComments":17671},"metadata":{"nodeName":"I Like Toast","xmppChat":false,"camo":{"' \
                  'markdown":false,"opengraph":false,"remotePods":false},"adminAccount":"podmin"}}' STATISTICS_JSON_DOC = '{"name":"diaspora*","network":"Diaspora","version":"0.5.7.0-p56ebcc76","registrations_open"' \
                      ':true,"services":[],"twitter":false,"tumblr":false,"facebook":false,"wordpress":false}' MASTODON_DOC = """
{"uri": "mastodon.local", "title": "Mastodon",
 "description": "This page describes the mastodon.local",
 "email": "hello@mastodon.local", "version": "2.4.0", "urls": {"streaming_api": "wss://mastodon.local"},
 "stats": {"user_count": 159726, "status_count": 6059606, "domain_count": 4703},
 "thumbnail": "https://files.mastodon.local/site_uploads/files/000/000/001/original/file.jpeg",
 "languages": ["en"],
 "contact_account": {"id": "1", "username": "Admin", "acct": "Admin", "display_name": "Admin dude", "locked": false,
                     "bot": false, "created_at": "2016-03-16T14:34:26.392Z",
                     "note": "\u003cp\u003eSuperuser\u003c/p\u003e",
                     "url": "https://mastodon.local/@Admin",
                     "avatar": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png",
                     "avatar_static": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png",
                     "header": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg",
                     "header_static": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg",
                     "followers_count": 81779, "following_count": 506, "statuses_count": 36722, "emojis": [],
                     "fields": []}}
""" MASTODON_ACTIVITY_DOC = """
    [
        {"week":"1526256000","statuses":"200229","logins":"10034","registrations":"1379"},
        {"week":"1526860800","statuses":"121188","logins":"8779","registrations":"1143"}
    ]
""" MASTODON_RC_DOC = """
{"uri": "mastodon.local", "title": "Mastodon",
 "description": "This page describes the mastodon.local",
 "email": "hello@mastodon.local", "version": "2.4.1rc1", "urls": {"streaming_api": "wss://mastodon.local"},
 "stats": {"user_count": 159726, "status_count": 6059606, "domain_count": 4703},
 "thumbnail": "https://files.mastodon.local/site_uploads/files/000/000/001/original/file.jpeg",
 "languages": ["en"],
 "contact_account": {"id": "1", "username": "Admin", "acct": "Admin", "display_name": "Admin dude", "locked": false,
                     "bot": false, "created_at": "2016-03-16T14:34:26.392Z",
                     "note": "\u003cp\u003eSuperuser\u003c/p\u003e",
                     "url": "https://mastodon.local/@Admin",
                     "avatar": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png",
                     "avatar_static": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png",
                     "header": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg",
                     "header_static": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg",
                     "followers_count": 81779, "following_count": 506, "statuses_count": 36722, "emojis": [],
                     "fields": []}}
""" MASTODON_DOC_NULL_CONTACT = """
{"uri": "mastodon.local", "title": "Mastodon",
 "description": "This page describes the mastodon.local",
 "email": "", "version": "2.4.0", "urls": {"streaming_api": "wss://mastodon.local"},
 "stats": {"user_count": 159726, "status_count": 6059606, "domain_count": 4703},
 "thumbnail": "https://files.mastodon.local/site_uploads/files/000/000/001/original/file.jpeg",
 "languages": ["en"],
 "contact_account": null}
""" MATRIX_SYNAPSE_DOC = """
    {"server": {"name": "Synapse", "version": "0.33.8"}}
""" PLEROMA_MASTODON_API_DOC = """
{"description":"A Pleroma instance, an alternative fediverse server","email":"foobar@pleroma.local","max_toot_chars":5000,"stats":{"domain_count":5,"status_count":127,"user_count":11},"thumbnail":"https://kemo.one/instance/thumbnail.jpeg","title":"Pleromaa","uri":"https://pleroma.local","urls":{"streaming_api":"wss://pleroma.local"},"version":"2.3.3 (compatible; Pleroma 0.9.0 ffa552f1a41c530a7eff25d27cf0a82e710067b6)"}
""" NODEINFO_21_DOC_INVALID_USAGE_COUNTS = \
                  '{"version":"2.1","software":{"name":"pleroma","version":"0.7.4.0-pd0313756"},"protocols":' \
                  '["activitypub"],"services":{"inbound":[],"outbound":[]},"openRegistrations"' \
                  ':true,"usage":{"users":{"total":348},"localPosts":{"all":3,"direct":1}},' \
                  '"metadata":{}}' MASTODON_DOC_3 = """
{"uri": "mastodon.local", "title": "Mastodon",
 "description": "This page describes the mastodon.local",
 "email": "hello@mastodon.local", "version": "3.0.0", "urls": {"streaming_api": "wss://mastodon.local"},
 "stats": {"user_count": 159726, "status_count": 6059606, "domain_count": 4703},
 "thumbnail": "https://files.mastodon.local/site_uploads/files/000/000/001/original/file.jpeg",
 "languages": ["en"],
 "contact_account": {"id": "1", "username": "Admin", "acct": "Admin", "display_name": "Admin dude", "locked": false,
                     "bot": false, "created_at": "2016-03-16T14:34:26.392Z",
                     "note": "\u003cp\u003eSuperuser\u003c/p\u003e",
                     "url": "https://mastodon.local/@Admin",
                     "avatar": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png",
                     "avatar_static": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png",
                     "header": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg",
                     "header_static": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg",
                     "followers_count": 81779, "following_count": 506, "statuses_count": 36722, "emojis": [],
                     "fields": []}}
"""
import json
from unittest.mock import patch
from federation.hostmeta.parsers import (
    parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document, int_or_none,
    parse_mastodon_document, parse_matrix_document)
from federation.tests.fixtures.hostmeta import (
    NODEINFO2_10_DOC, NODEINFO_10_DOC, NODEINFO_20_DOC, STATISTICS_JSON_DOC, MASTODON_DOC, MASTODON_ACTIVITY_DOC,
    MASTODON_RC_DOC, MASTODON_DOC_NULL_CONTACT, MATRIX_SYNAPSE_DOC, PLEROMA_MASTODON_API_DOC,
    NODEINFO_21_DOC_INVALID_USAGE_COUNTS, MASTODON_DOC_3)



class TestIntOrNone:
    def test_returns_negative_values_as_none(self):
        assert int_or_none(-1) is None


class TestParseMastodonDocument:
    @patch('federation.hostmeta.fetchers.fetch_nodeinfo_document', autospec=True)
    def test_calls_nodeinfo_fetcher_if_pleroma(self, mock_fetch):

        parse_mastodon_document(json.loads(PLEROMA_MASTODON_API_DOC), 'example.com')