commit f4a34c2998f44a7142542ed1f6e5818524ebc292 Author: Clay Gerrard Date: Thu Oct 1 11:16:04 2020 -0500 Log storage policy with s3api requests Change-Id: I5fe5ab31d6b2d9f7b6ecb3bfa246433a78e54808 diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index ad72272..01d8f4a 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -1350,6 +1350,10 @@ class S3Request(swob.Request): # in pipeline that need Swift PATH_INFO like ceilometermiddleware. self.environ['s3api.backend_path'] = \ sw_resp.environ['PATH_INFO'] + # Propogate backend headers back into our req headers for logging + for k, v in sw_req.headers.items(): + if k.lower().startswith('x-backend-'): + self.headers.setdefault(k, v) resp = S3Response.from_swift_resp(sw_resp) status = resp.status_int # pylint: disable-msg=E1101 diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index e9a2090..f299aa6 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -171,6 +171,10 @@ class FakeSwift(object): k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data + # simulate object GET/HEAD + elif method in ('GET', 'HEAD') and obj: + req.headers['X-Backend-Storage-Policy-Index'] = '2' + # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( diff --git a/test/unit/common/middleware/s3api/__init__.py b/test/unit/common/middleware/s3api/__init__.py index 007dda7..93e210a 100644 --- a/test/unit/common/middleware/s3api/__init__.py +++ b/test/unit/common/middleware/s3api/__init__.py @@ -93,7 +93,7 @@ class S3ApiTestCase(unittest.TestCase): self.app = FakeApp() self.swift = self.app.swift self.s3api = filter_factory({}, **self.conf)(self.app) - self.s3api.logger = self.swift.logger = debug_logger() + self.logger = self.s3api.logger = self.swift.logger = debug_logger() self.swift.register('HEAD', '/v1/AUTH_test', swob.HTTPOk, {}, None) diff --git a/test/unit/common/middleware/s3api/test_obj.py b/test/unit/common/middleware/s3api/test_obj.py index 4c1f5ff..0e2aa7b 100644 --- a/test/unit/common/middleware/s3api/test_obj.py +++ b/test/unit/common/middleware/s3api/test_obj.py @@ -26,6 +26,7 @@ import json from swift.common import swob from swift.common.swob import Request +from swift.common.middleware.proxy_logging import ProxyLoggingMiddleware from test.unit.common.middleware.s3api import S3ApiTestCase from test.unit.common.middleware.s3api.test_s3_acl import s3acl @@ -77,6 +78,8 @@ class TestS3ApiObj(S3ApiTestCase): 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') + # we'll want this for logging + self.assertEqual(req.headers['X-Backend-Storage-Policy-Index'], '2') unexpected_headers = [] for key, val in self.response_headers.items(): @@ -173,6 +176,19 @@ class TestS3ApiObj(S3ApiTestCase): def test_object_HEAD(self): self._test_object_GETorHEAD('HEAD') + def test_object_policy_index_logging(self): + req = Request.blank('/bucket/object', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + self.s3api = ProxyLoggingMiddleware(self.s3api, {}, logger=self.logger) + status, headers, body = self.call_s3api(req) + access_lines = self.logger.get_lines_for_level('info') + self.assertEqual(1, len(access_lines)) + parts = access_lines[0].split() + self.assertEqual(' '.join(parts[3:7]), + 'GET /bucket/object HTTP/1.0 200') + self.assertEqual(parts[-1], '2') + def _test_object_HEAD_Range(self, range_value): req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'HEAD'}, diff --git a/test/unit/common/middleware/test_symlink.py b/test/unit/common/middleware/test_symlink.py index a5e6cba..c8d6b82 100644 --- a/test/unit/common/middleware/test_symlink.py +++ b/test/unit/common/middleware/test_symlink.py @@ -438,9 +438,12 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase): self.assertNotIn('X-Symlink-Target-Account', dict(headers)) self.assertIn(('Content-Location', '/v1/a2/c1/o'), headers) calls = self.app.calls_with_headers - req_headers['Host'] = 'localhost:80' - req_headers['X-Backend-Ignore-Range-If-Metadata-Present'] = \ - 'x-object-sysmeta-symlink-target' + req_headers.update({ + 'Host': 'localhost:80', + 'X-Backend-Ignore-Range-If-Metadata-Present': + 'x-object-sysmeta-symlink-target', + 'X-Backend-Storage-Policy-Index': '2', + }) self.assertEqual(req_headers, calls[0].headers) req_headers['User-Agent'] = 'Swift' self.assertEqual(req_headers, calls[1].headers) @@ -611,9 +614,12 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase): self.assertIn(('X-Object-Meta-Color', 'Green'), headers) self.assertIn(('Content-Location', '/v1/a2/c1/o'), headers) calls = self.app.calls_with_headers - req_headers['Host'] = 'localhost:80' - req_headers['X-Backend-Ignore-Range-If-Metadata-Present'] = \ - 'x-object-sysmeta-symlink-target' + req_headers.update({ + 'Host': 'localhost:80', + 'X-Backend-Ignore-Range-If-Metadata-Present': + 'x-object-sysmeta-symlink-target', + 'X-Backend-Storage-Policy-Index': '2', + }) self.assertEqual(req_headers, calls[0].headers) req_headers['User-Agent'] = 'Swift' self.assertEqual(req_headers, calls[1].headers) diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index a57eae0..dd26d5f 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -1315,6 +1315,7 @@ class TestInternalClient(unittest.TestCase): 'host': 'localhost:80', # from swob.Request.blank 'user-agent': 'test', # from InternalClient.make_request 'x-backend-allow-reserved-names': 'true', # also from IC + 'x-backend-storage-policy-index': '2', # from proxy-server app }) self.assertEqual(app.calls_with_headers, [( 'GET', path_info + '?symlink=get', HeaderKeyDict(req_headers))])