Merge PR #57215 into main

* refs/pull/57215/head:
	doc: document new --output-file switch
	test/cli: ignore tmp_file_template
	qa/workunits: add --output-file test in cephtool workunit
	common,ceph: add output file switch to dump json to
	common/options: add configs for temporary files made by daemons
	common/Formatter: write the pending string on flush

Reviewed-by: Leonid Usov <leonid.usov@ibm.com>
Reviewed-by: Anthony D Atri <anthony.datri@gmail.com>

10 files changed, 201 insertions, 9 deletions

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 993fafd0ac9..843302e08ae 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -1,5 +1,12 @@
 >=19.0.0
 
+* ceph: a new --daemon-output-file switch is available for `ceph tell` commands
+  to dump output to a file local to the daemon. For commands which produce
+  large amounts of output, this avoids a potential spike in memory usage on the
+  daemon, allows for faster streaming writes to a file local to the daemon, and
+  reduces time holding any locks required to execute the command. For analysis,
+  it is necessary to retrieve the file from the host running the daemon
+  manually. Currently, only --format=json|json-pretty are supported.
 * RGW: GetObject and HeadObject requests now return a x-rgw-replicated-at
   header for replicated objects. This timestamp can be compared against the
   Last-Modified header to determine how long the object took to replicate.
diff --git a/doc/man/8/ceph.rst b/doc/man/8/ceph.rst
index 478d3c6b3ce..bcc63b871e0 100644
--- a/doc/man/8/ceph.rst
+++ b/doc/man/8/ceph.rst
@@ -1724,7 +1724,25 @@ Options
 
 .. option:: -f {json,json-pretty,xml,xml-pretty,plain,yaml}, --format
 
-	Format of output. Note: yaml is only valid for orch commands. 
+	Format of output.
+
+    Note: yaml is only valid for orch commands.
+
+.. option:: --daemon-output-file OUTPUT_FILE
+
+    When using --format=json|json-pretty, you may specify a file name on the
+    host running the daemon to stream output to. Be mindful this is probably
+    not the same machine running the ceph command. So to analyze the output, it
+    will be necessary to fetch the file once the command completes.
+
+    OUTPUT_FILE may also be ``:tmp:``, indicating that the daemon should create
+    a temporary file (subject to configurations tmp_dir and tmp_file_template).
+
+    The ``tell`` command will output json with the path to the output file
+    written to, the size of the file, the result code of the command, and any
+    output produced by the command.
+
+    Note: this option is only used for ``ceph tell`` commands.
 
 .. option:: --connect-timeout CLUSTER_TIMEOUT
 
diff --git a/doc/rados/configuration/common.rst b/doc/rados/configuration/common.rst
index c397f4e52ac..0c8350efb52 100644
--- a/doc/rados/configuration/common.rst
+++ b/doc/rados/configuration/common.rst
@@ -44,6 +44,27 @@ For more about configuring a network for use with Ceph, see the `Network
 Configuration Reference`_ .
 
 
+Temporary Directory
+===================
+
+Some operations will cause a daemon to write to a temporary file. These files
+are located according to the ``tmp_dir`` config.
+
+.. confval:: tmp_dir
+
+The ``$TMPDIR`` environment variable is used to initialize the config, if
+present, but may be overriden on the command-line. A default may also
+be set for the cluster using the usual ``ceph config`` API.
+
+The template for the temporary files created by daemons is controlled
+by the ``tmp_file_template`` config.
+
+.. confval:: tmp_file_template
+
+One example where temporary files are created by daemons is the use of the
+``--daemon-output-file=:tmp:`` argument to the ``ceph tell`` command.
+
+
 Monitors
 ========
 
diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index 6dc2ee9513c..8a7ab1ea781 100755
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -855,6 +855,47 @@ function without_test_dup_command()
   fi
 }
 
+function test_tell_output_file()
+{
+  name="$1"
+  shift
+
+  # Test --daemon-output-file
+  # N.B.: note this only works if $name is on the same node as this script!
+  J=$(ceph tell --format=json --daemon-output-file=/tmp/foo "$name" version)
+  expect_true jq -e '.path == "/tmp/foo"' <<<"$J"
+  expect_true test -e /tmp/foo
+  # only one line of json
+  expect_true sed '2q1' < /tmp/foo > /dev/null
+  expect_true jq -e '.version | length > 0' < /tmp/foo
+  rm -f /tmp/foo
+
+  J=$(ceph tell --format=json-pretty --daemon-output-file=/tmp/foo "$name" version)
+  expect_true jq -e '.path == "/tmp/foo"' <<<"$J"
+  expect_true test -e /tmp/foo
+  # more than one line of json
+  expect_false sed '2q1' < /tmp/foo > /dev/null
+  expect_true jq -e '.version | length > 0' < /tmp/foo
+  rm -f /tmp/foo
+
+  # Test --daemon-output-file=:tmp:
+  J=$(ceph tell --format=json --daemon-output-file=":tmp:" "$name" version)
+  path=$(jq -r .path <<<"$J")
+  expect_true test -e "$path"
+  # only one line of json
+  expect_true sed '2q1' < "$path" > /dev/null
+  expect_true jq -e '.version | length > 0' < "$path"
+  rm -f "$path"
+
+  J=$(ceph tell --format=json-pretty --daemon-output-file=":tmp:" "$name" version)
+  path=$(jq -r .path <<<"$J")
+  expect_true test -e "$path"
+  # only one line of json
+  expect_false sed '2q1' < "$path" > /dev/null
+  expect_true jq -e '.version | length > 0' < "$path"
+  rm -f "$path"
+}
+
 function test_mds_tell()
 {
   local FS_NAME=cephfs
@@ -896,6 +937,8 @@ function test_mds_tell()
   done
   echo New GIDs: $new_mds_gids
 
+  test_tell_output_file mds."$FS_NAME":0
+
   remove_all_fs
   ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it
   ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it
@@ -2628,6 +2671,8 @@ function test_mon_tell()
     ceph_watch_wait "${m} \[DBG\] from.*cmd='sessions' args=\[\]: dispatch"
   done
   expect_false ceph tell mon.foo version
+
+  test_tell_output_file mon.0
 }
 
 function test_mon_ping()
diff --git a/src/ceph.in b/src/ceph.in
index 11a76511a8e..51743dd9ae8 100755
--- a/src/ceph.in
+++ b/src/ceph.in
@@ -336,6 +336,8 @@ def parse_cmdargs(args=None, target='') -> Tuple[argparse.ArgumentParser,
     parser.add_argument('--concise', dest='verbose', action="store_false",
                         help="make less verbose")
 
+    parser.add_argument('--daemon-output-file', dest='daemon_output_file',
+                        help="output file location local to the daemon for JSON produced by tell commands")
     parser.add_argument('-f', '--format', choices=['json', 'json-pretty',
                         'xml', 'xml-pretty', 'plain', 'yaml'],
                         help="Note: yaml is only valid for orch commands", dest='output_format')
@@ -580,6 +582,8 @@ def do_command(parsed_args, target, cmdargs, sigdict, inbuf, verbose):
     if valid_dict:
         if parsed_args.output_format:
             valid_dict['format'] = parsed_args.output_format
+        if parsed_args.daemon_output_file:
+            valid_dict['output-file'] = parsed_args.daemon_output_file
         if verbose:
             print("Submitting command: ", valid_dict, file=sys.stderr)
     else:
diff --git a/src/common/Formatter.h b/src/common/Formatter.h
index 65ffb0a6855..3e94c687114 100644
--- a/src/common/Formatter.h
+++ b/src/common/Formatter.h
@@ -200,7 +200,7 @@ namespace ceph {
     int get_len() const override;
     void write_raw_data(const char *data) override;
 
-  protected:
+protected:
     virtual bool handle_value(std::string_view name, std::string_view s, bool quoted) {
       return false; /* is handling done? */
     }
@@ -219,8 +219,9 @@ namespace ceph {
       return m_ss;
     }
 
-  private:
+    void finish_pending_string();
 
+private:
     struct json_formatter_stack_entry_d {
       int size = 0;
       bool is_array = false;
@@ -231,7 +232,6 @@ namespace ceph {
     void print_quoted_string(std::string_view s);
     void print_name(std::string_view name);
     void print_comma(json_formatter_stack_entry_d& entry);
-    void finish_pending_string();
 
     template <class T>
     void add_value(std::string_view name, T val);
@@ -254,6 +254,14 @@ public:
     {
     }
     ~JSONFormatterFile() {
+      flush();
+    }
+
+    void flush(std::ostream& os) override {
+      flush();
+    }
+    void flush() {
+      JSONFormatter::finish_pending_string();
       file.flush();
     }
 
@@ -264,6 +272,9 @@ public:
     int get_len() const override {
       return file.tellp();
     }
+    std::ofstream const& get_ofstream() const {
+      return file;
+    }
 
 protected:
     std::ostream& get_ss() override {
diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc
index 343cb441417..78eb8e45eb3 100644
--- a/src/common/admin_socket.cc
+++ b/src/common/admin_socket.cc
@@ -15,6 +15,8 @@
 #include <sys/un.h>
 #include <optional>
 
+#include <stdlib.h>
+
 #include "common/admin_socket.h"
 #include "common/admin_socket_client.h"
 #include "common/dout.h"
@@ -506,7 +508,46 @@ void AdminSocket::execute_command(
 		     empty);
   }
 
-  auto f = Formatter::create(format, "json-pretty", "json-pretty");
+  ldout(m_cct, 20) << __func__ << ": format is " << format << " prefix is " << prefix << dendl;
+
+  string output;
+  try {
+    cmd_getval(cmdmap, "output-file", output);
+    if (!output.empty()) {
+      ldout(m_cct, 20) << __func__ << ": output file is " << output << dendl;
+    }
+  } catch (const bad_cmd_get& e) {
+    output = "";
+  }
+
+  if (output == ":tmp:") {
+    auto path = m_cct->_conf.get_val<std::string>("tmp_file_template");
+    if (int fd = mkstemp(path.data()); fd >= 0) {
+      close(fd);
+      output = path;
+      ldout(m_cct, 20) << __func__ << ": output file created in tmp_dir is " << output << dendl;
+    } else {
+      return on_finish(-errno, "temporary output file could not be opened", empty);
+    }
+  }
+
+  Formatter* f;
+  if (!output.empty()) {
+    if (!(format == "json" || format == "json-pretty")) {
+      return on_finish(-EINVAL, "unsupported format for --output-file", empty);
+    }
+    ldout(m_cct, 10) << __func__ << ": opening file for json output: " << output << dendl;
+    bool pretty = (format == "json-pretty");
+    auto* jff = new JSONFormatterFile(output, pretty);
+    auto&& of = jff->get_ofstream();
+    if (!of.is_open()) {
+      delete jff;
+      return on_finish(-EIO, "output file could not be opened", empty);
+    }
+    f = jff;
+  } else {
+    f = Formatter::create(format, "json-pretty", "json-pretty");
+  }
 
   auto [retval, hook] = find_matched_hook(prefix, cmdmap);
   switch (retval) {
@@ -524,10 +565,27 @@ void AdminSocket::execute_command(
 
   hook->call_async(
     prefix, cmdmap, f, inbl,
-    [f, on_finish](int r, const std::string& err, bufferlist& out) {
+    [f, output, on_finish, m_cct=m_cct](int r, const std::string& err, bufferlist& out) {
       // handle either existing output in bufferlist *or* via formatter
-      if (r >= 0 && out.length() == 0) {
-	f->flush(out);
+      ldout(m_cct, 10) << __func__ << ": command completed with result " << r << dendl;
+      if (auto* jff = dynamic_cast<JSONFormatterFile*>(f); jff != nullptr) {
+        ldout(m_cct, 25) << __func__ << ": flushing file" << dendl;
+        jff->flush();
+        auto* outf = new JSONFormatter(true);
+        outf->open_object_section("result");
+        outf->dump_string("path", output);
+        outf->dump_int("result", r);
+        outf->dump_string("output", out.to_str());
+        outf->dump_int("len", jff->get_len());
+        outf->close_section();
+        CachedStackStringStream css;
+        outf->flush(*css);
+        delete outf;
+        out.clear();
+        out.append(css->strv());
+      } else if (r >= 0 && out.length() == 0) {
+        ldout(m_cct, 25) << __func__ << ": out is empty, dumping formatter" << dendl;
+        f->flush(out);
       }
       delete f;
       on_finish(r, err, out);
diff --git a/src/common/config.cc b/src/common/config.cc
index c8101587b71..604b3c35d5e 100644
--- a/src/common/config.cc
+++ b/src/common/config.cc
@@ -493,6 +493,11 @@ void md_config_t::parse_env(unsigned entity_type,
     }
   }
 
+  if (auto s = getenv("TMPDIR"); s) {
+    string err;
+    _set_val(values, tracker, s, *find_option("tmp_dir"), CONF_ENV, &err);
+  }
+
   // Apply pod memory limits:
   //
   // There are two types of resource requests: `limits` and `requests`.
diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 52d306e4c11..7366bbb31c6 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -250,6 +250,29 @@ options:
   flags:
   - startup
   with_legacy: true
+- name: tmp_dir
+  type: str
+  level: advanced
+  desc: path for the 'tmp' directory
+  default: /tmp
+  services:
+  - common
+  see_also:
+  - admin_socket
+  flags:
+  - runtime
+- name: tmp_file_template
+  type: str
+  level: advanced
+  desc: Template for temporary files created by daemons for ceph tell commands
+  long_desc: The template file name prefix for temporary files. For example, temporary files may be created by `ceph tell` commands using the --daemon-output-file switch.
+  daemon_default: $tmp_dir/$cluster-$name.XXXXXX
+  services:
+  - osd
+  - mds
+  - mon
+  flags:
+  - runtime
 - name: admin_socket
   type: str
   level: advanced
diff --git a/src/test/cli/ceph-conf/show-config.t b/src/test/cli/ceph-conf/show-config.t
index 45405e4b7a0..3e7a6fcf911 100644
--- a/src/test/cli/ceph-conf/show-config.t
+++ b/src/test/cli/ceph-conf/show-config.t
@@ -1,4 +1,4 @@
-  $ ceph-conf -n osd.0 --show-config -c /dev/null | grep ceph-osd
+  $ ceph-conf -n osd.0 --show-config -c /dev/null | grep ceph-osd | grep -v tmp_file_template
   admin_socket = /var/run/ceph/ceph-osd.0.asok
   log_file = /var/log/ceph/ceph-osd.0.log
   mon_debug_dump_location = /var/log/ceph/ceph-osd.0.tdump