UTF-8 Content Negotiation

Signed-off-by: Owen Williams <[email protected]>

Author: ywwg

prometheus_client/__init__.py

@@ -5,7 +5,7 @@
     process_collector, registry,
 )
 from .exposition import (
-    CONTENT_TYPE_LATEST, delete_from_gateway, generate_latest,
+    CONTENT_TYPE_PLAIN, delete_from_gateway, generate_latest,
     instance_ip_grouping_key, make_asgi_app, make_wsgi_app, MetricsHandler,
     push_to_gateway, pushadd_to_gateway, start_http_server, start_wsgi_server,
     write_to_textfile,
@@ -32,7 +32,7 @@
     'Enum',
     'enable_created_metrics',
     'disable_created_metrics',
-    'CONTENT_TYPE_LATEST',
+    'CONTENT_TYPE_PLAIN',
     'generate_latest',
     'MetricsHandler',
     'make_wsgi_app',

prometheus_client/exposition.py

@@ -23,7 +23,7 @@
 from .validation import _is_valid_legacy_metric_name
 
 __all__ = (
-    'CONTENT_TYPE_LATEST',
+    'CONTENT_TYPE_PLAIN',
     'delete_from_gateway',
     'generate_latest',
     'instance_ip_grouping_key',
@@ -37,7 +37,7 @@
     'write_to_textfile',
 )
 
-CONTENT_TYPE_LATEST = 'text/plain; version=0.0.4; charset=utf-8'
+CONTENT_TYPE_PLAIN = 'text/plain; version=0.0.4; charset=utf-8'
 """Content type of the latest text format"""
 
 
@@ -245,29 +245,38 @@ class TmpServer(ThreadingWSGIServer):
 start_http_server = start_wsgi_server
 
 
-def generate_latest(registry: CollectorRegistry = REGISTRY) -> bytes:
-    """Returns the metrics from the registry in latest text format as a string."""
+def generate_latest(registry: CollectorRegistry = REGISTRY, escaping: str = openmetrics.UNDERSCORES) -> bytes:
+    """
+    Generates the exposition format using the basic Prometheus text format.
+
+    Params:
+        registry: CollectorRegistry to export data from.
+        escaping: Escaping scheme used for metric and label names.
+    
+    Returns: UTF-8 encoded string containing the metrics in text format.
+    """
 
     def sample_line(samples):
         if samples.labels:
             labelstr = '{0}'.format(','.join(
+                # Label values always support UTF-8
                 ['{}="{}"'.format(
-                    openmetrics.escape_label_name(k), openmetrics._escape(v))
+                    openmetrics.escape_label_name(k, escaping), openmetrics._escape(v, openmetrics.ALLOWUTF8, False))
                     for k, v in sorted(samples.labels.items())]))
         else:
             labelstr = ''
         timestamp = ''
         if samples.timestamp is not None:
             # Convert to milliseconds.
             timestamp = f' {int(float(samples.timestamp) * 1000):d}'
-        if _is_valid_legacy_metric_name(samples.name):
+        if escaping != openmetrics.ALLOWUTF8 or _is_valid_legacy_metric_name(samples.name):
             if labelstr:
                 labelstr = '{{{0}}}'.format(labelstr)
-            return f'{samples.name}{labelstr} {floatToGoString(samples.value)}{timestamp}\n'
+            return f'{openmetrics.escape_metric_name(samples.name, escaping)}{labelstr} {floatToGoString(samples.value)}{timestamp}\n'
         maybe_comma = ''
         if labelstr:
             maybe_comma = ','
-        return f'{{{openmetrics.escape_metric_name(samples.name)}{maybe_comma}{labelstr}}} {floatToGoString(samples.value)}{timestamp}\n'
+        return f'{{{openmetrics.escape_metric_name(samples.name, escaping)}{maybe_comma}{labelstr}}} {floatToGoString(samples.value)}{timestamp}\n'
 
     output = []
     for metric in registry.collect():
@@ -290,8 +299,8 @@ def sample_line(samples):
                 mtype = 'untyped'
 
             output.append('# HELP {} {}\n'.format(
-                openmetrics.escape_metric_name(mname), metric.documentation.replace('\\', r'\\').replace('\n', r'\n')))
-            output.append(f'# TYPE {openmetrics.escape_metric_name(mname)} {mtype}\n')
+                openmetrics.escape_metric_name(mname, escaping), metric.documentation.replace('\\', r'\\').replace('\n', r'\n')))
+            output.append(f'# TYPE {openmetrics.escape_metric_name(mname, escaping)} {mtype}\n')
 
             om_samples: Dict[str, List[str]] = {}
             for s in metric.samples:
@@ -307,21 +316,69 @@ def sample_line(samples):
             raise
 
         for suffix, lines in sorted(om_samples.items()):
-            output.append('# HELP {} {}\n'.format(openmetrics.escape_metric_name(metric.name + suffix),
+            output.append('# HELP {} {}\n'.format(openmetrics.escape_metric_name(metric.name + suffix, escaping),
                                                   metric.documentation.replace('\\', r'\\').replace('\n', r'\n')))
-            output.append(f'# TYPE {openmetrics.escape_metric_name(metric.name + suffix)} gauge\n')
+            output.append(f'# TYPE {openmetrics.escape_metric_name(metric.name + suffix, escaping)} gauge\n')
             output.extend(lines)
     return ''.join(output).encode('utf-8')
 
 
 def choose_encoder(accept_header: str) -> Tuple[Callable[[CollectorRegistry], bytes], str]:
+    # Python client library accepts a much narrower range of content-types than
+    # Prometheus does -- UTF-8 is only supported on OpenMetrics v1.0.0.
     accept_header = accept_header or ''
+    escaping = openmetrics.UNDERSCORES
     for accepted in accept_header.split(','):
         if accepted.split(';')[0].strip() == 'application/openmetrics-text':
-            return (openmetrics.generate_latest,
-                    openmetrics.CONTENT_TYPE_LATEST)
-    return generate_latest, CONTENT_TYPE_LATEST
-
+            toks = accepted.split(';')
+            version = _get_version(toks)
+            escaping = _get_escaping(toks)
+            # Only return an escaping header if we have a good version and
+            # mimetype.
+            if version == '1.0.0':
+                return (openmetrics.generate_latest_fn(escaping), 
+                        openmetrics.CONTENT_TYPE_LATEST + '; escaping=' + str(escaping))
+    return generate_latest, CONTENT_TYPE_PLAIN
+
+
+def _get_version(accept_header: List[str]) -> str:
+    """Return the version tag from the Accept header.
+
+    If no escaping scheme is specified, returns empty string."""
+    
+    for tok in accept_header:
+        if '=' not in tok:
+            continue
+        key, value = tok.strip().split('=', 1)
+        if key == 'version':
+            return value
+    return ""
+            
+
+def _get_escaping(accept_header: List[str]) -> str:
+    """Return the escaping scheme from the Accept header.
+
+    If no escaping scheme is specified or the scheme is not one of the allowed
+    strings, defaults to UNDERSCORES."""
+    
+    for tok in accept_header:
+        if '=' not in tok:
+            continue
+        key, value = tok.strip().split('=', 1)
+        if key != 'escaping':
+            continue
+        if value == openmetrics.ALLOWUTF8:
+            return openmetrics.ALLOWUTF8
+        elif value == openmetrics.UNDERSCORES:
+            return openmetrics.UNDERSCORES
+        elif value == openmetrics.DOTS:
+            return openmetrics.DOTS
+        elif value == openmetrics.VALUES:
+            return openmetrics.VALUES
+        else:
+            return openmetrics.UNDERSCORES
+    return openmetrics.UNDERSCORES
+                
 
 def gzip_accepted(accept_encoding_header: str) -> bool:
     accept_encoding_header = accept_encoding_header or ''
@@ -369,15 +426,15 @@ def factory(cls, registry: CollectorRegistry) -> type:
         return MyMetricsHandler
 
 
-def write_to_textfile(path: str, registry: CollectorRegistry) -> None:
+def write_to_textfile(path: str, registry: CollectorRegistry, escaping: str = openmetrics.ALLOWUTF8) -> None:
     """Write metrics to the given path.
 
     This is intended for use with the Node exporter textfile collector.
     The path must end in .prom for the textfile collector to process it."""
     tmppath = f'{path}.{os.getpid()}.{threading.current_thread().ident}'
     try:
         with open(tmppath, 'wb') as f:
-            f.write(generate_latest(registry))
+            f.write(generate_latest(registry, escaping))
 
         # rename(2) is atomic but fails on Windows if the destination file exists
         if os.name == 'nt':
@@ -645,7 +702,7 @@ def _use_gateway(
 
     handler(
         url=url, method=method, timeout=timeout,
-        headers=[('Content-Type', CONTENT_TYPE_LATEST)], data=data,
+        headers=[('Content-Type', CONTENT_TYPE_PLAIN)], data=data,
     )()
 
 

prometheus_client/openmetrics/exposition.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from io import StringIO
+from sys import maxunicode
 
 from ..utils import floatToGoString
 from ..validation import (
@@ -8,6 +10,13 @@
 
 CONTENT_TYPE_LATEST = 'application/openmetrics-text; version=1.0.0; charset=utf-8'
 """Content type of the latest OpenMetrics text format"""
+ESCAPING_HEADER_TAG = 'escaping'
+
+
+ALLOWUTF8 = 'allow-utf-8'
+UNDERSCORES = 'underscores'
+DOTS = 'dots'
+VALUES = 'values'
 
 
 def _is_valid_exemplar_metric(metric, sample):
@@ -20,30 +29,36 @@ def _is_valid_exemplar_metric(metric, sample):
     return False
 
 
-def generate_latest(registry):
+def generate_latest_fn(escaping):
+    '''Returns a generate_latest function that will always use the given escaping.'''
+    return lambda registry: generate_latest(registry, escaping)
+    
+
+def generate_latest(registry, escaping):
     '''Returns the metrics from the registry in latest text format as a string.'''
     output = []
     for metric in registry.collect():
         try:
             mname = metric.name
             output.append('# HELP {} {}\n'.format(
-                escape_metric_name(mname), _escape(metric.documentation)))
-            output.append(f'# TYPE {escape_metric_name(mname)} {metric.type}\n')
+                escape_metric_name(mname, escaping), _escape(metric.documentation, ALLOWUTF8, False)))
+            output.append(f'# TYPE {escape_metric_name(mname, escaping)} {metric.type}\n')
             if metric.unit:
-                output.append(f'# UNIT {escape_metric_name(mname)} {metric.unit}\n')
+                output.append(f'# UNIT {escape_metric_name(mname, escaping)} {metric.unit}\n')
             for s in metric.samples:
-                if not _is_valid_legacy_metric_name(s.name):
-                    labelstr = escape_metric_name(s.name)
+                if escaping == ALLOWUTF8 and not _is_valid_legacy_metric_name(s.name):
+                    labelstr = escape_metric_name(s.name, escaping)
                     if s.labels:
                         labelstr += ', '
                 else:
                     labelstr = ''
 
                 if s.labels:
                     items = sorted(s.labels.items())
+                    # Label values always support UTF-8
                     labelstr += ','.join(
                         ['{}="{}"'.format(
-                            escape_label_name(k), _escape(v))
+                            escape_label_name(k, escaping), _escape(v, ALLOWUTF8, False))
                             for k, v in items])
                 if labelstr:
                     labelstr = "{" + labelstr + "}"
@@ -71,9 +86,9 @@ def generate_latest(registry):
                 timestamp = ''
                 if s.timestamp is not None:
                     timestamp = f' {s.timestamp}'
-                if _is_valid_legacy_metric_name(s.name):
+                if (escaping != ALLOWUTF8) or _is_valid_legacy_metric_name(s.name):
                     output.append('{}{} {}{}{}\n'.format(
-                        s.name,
+                        _escape(s.name, escaping, False),
                         labelstr,
                         floatToGoString(s.value),
                         timestamp,
@@ -94,24 +109,114 @@ def generate_latest(registry):
     return ''.join(output).encode('utf-8')
 
 
-def escape_metric_name(s: str) -> str:
+def escape_metric_name(s: str, escaping: str) -> str:
     """Escapes the metric name and puts it in quotes iff the name does not
     conform to the legacy Prometheus character set.
     """
-    if _is_valid_legacy_metric_name(s):
+    if len(s) == 0:
         return s
-    return '"{}"'.format(_escape(s))
+    if escaping == ALLOWUTF8:
+        if not _is_valid_legacy_metric_name(s):
+            return '"{}"'.format(_escape(s, escaping, False))
+        return _escape(s, escaping, False)
+    elif escaping == UNDERSCORES:
+        if _is_valid_legacy_metric_name(s):
+            return s
+        return _escape(s, escaping, False)
+    elif escaping == DOTS:
+        return _escape(s, escaping, False)
+    elif escaping == VALUES:
+        if _is_valid_legacy_metric_name(s):
+            return s
+        return _escape(s, escaping, False)
+    return s
 
 
-def escape_label_name(s: str) -> str:
+def escape_label_name(s: str, escaping: str) -> str:
     """Escapes the label name and puts it in quotes iff the name does not
     conform to the legacy Prometheus character set.
     """
-    if _is_valid_legacy_labelname(s):
+    if len(s) == 0:
         return s
-    return '"{}"'.format(_escape(s))
+    if escaping == ALLOWUTF8:
+        if not _is_valid_legacy_labelname(s):
+            return '"{}"'.format(_escape(s, escaping, True))
+        return _escape(s, escaping, True)
+    elif escaping == UNDERSCORES:
+        if _is_valid_legacy_labelname(s):
+            return s
+        return _escape(s, escaping, True)
+    elif escaping == DOTS:
+        return _escape(s, escaping, True)
+    elif escaping == VALUES:
+        if _is_valid_legacy_labelname(s):
+            return s
+        return _escape(s, escaping, True)
+    return s
 
 
-def _escape(s: str) -> str:
+def _escape(s: str, escaping: str, is_labelname: bool) -> str:
     """Performs backslash escaping on backslash, newline, and double-quote characters."""
-    return s.replace('\\', r'\\').replace('\n', r'\n').replace('"', r'\"')
+    if escaping == ALLOWUTF8:
+        return s.replace('\\', r'\\').replace('\n', r'\n').replace('"', r'\"')
+    elif escaping == UNDERSCORES:
+        escaped = StringIO()
+        for i, b in enumerate(s):
+            if _is_valid_legacy_rune(b, i, is_labelname):
+                escaped.write(b)
+            else:
+                escaped.write('_')
+        return escaped.getvalue()
+    elif escaping == DOTS:
+        escaped = StringIO()
+        for i, b in enumerate(s):
+            if b == '_':
+                escaped.write('__')
+            elif b == '.':
+                escaped.write('_dot_')
+            elif _is_valid_legacy_rune(b, i, is_labelname):
+                escaped.write(b)
+            else:
+                escaped.write('__')
+        return escaped.getvalue()
+    elif escaping == VALUES:
+        escaped = StringIO()
+        escaped.write("U__")
+        for i, b in enumerate(s):
+            if b == '_':
+                escaped.write("__")
+            elif _is_valid_legacy_rune(b, i, is_labelname):
+                escaped.write(b)
+            elif not _is_valid_utf8(b):
+                escaped.write("_FFFD_")
+            else:
+                escaped.write('_')
+                escaped.write(format(ord(b), 'x'))
+                escaped.write('_')
+        return escaped.getvalue()
+    return s
+
+
+def _is_valid_legacy_rune(b: str, i: int, is_labelname: bool) -> bool:
+    if len(b) != 1:
+        raise ValueError("Input 'b' must be a single character.")
+    if (
+        ('a' <= b <= 'z')
+        or ('A' <= b <= 'Z')
+        or (b == '_')
+        or ('0' <= b <= '9' and i > 0)
+    ):
+        return True
+    return not is_labelname and b == ':'
+
+
+_SURROGATE_MIN = 0xD800
+_SURROGATE_MAX = 0xDFFF
+
+
+def _is_valid_utf8(s: str) -> bool:
+    if 0 <= ord(s) < _SURROGATE_MIN:
+        return True
+    if _SURROGATE_MAX < ord(s) <= maxunicode:
+        return True
+    return False