Subject: [nfsv4] [PATCH 0/3] Introduce LAYOUTERROR and LAYOUTSTAT

From: Thomas Haynes <[email-masked]>
Date: Thu, 15 May 2014 17:56:06 -0700

As per the WG ML discussion, I am proposing LAYOUTERROR and
LAYOUTSTAT to provide generic errors and statistics for outstanding
layouts.

---

From: Thomas Haynes <[email-masked]>
Date: Thu, 15 May 2014 17:56:07 -0700

From: Tom Haynes <[email-masked]>

Signed-off-by: Tom Haynes <[email-masked]>
---
 Makefile                         |  4 ++++
 dotx.d/Makefile                  | 23 +++++++++++++----------
 dotx.d/layoutstats_args.x        | 10 ++++++++++
 dotx.d/layoutstats_res.x         |  5 +++++
 dotx.d/type_io_info.x            |  4 ++++
 nfsv42_middle_errors.xml         | 31 +++++++++++++++++++++++++++++++
 nfsv42_middle_new_errors.xml     | 31 +++++++++++++++++++++++++++++++
 nfsv42_middle_op_layoutstats.xml | 38 ++++++++++++++++++++++++++++++++++++++
 nfsv42_middle_op_mandlist.xml    |  1 +
 9 files changed, 137 insertions(+), 10 deletions(-)
 create mode 100644 dotx.d/layoutstats_args.x
 create mode 100644 dotx.d/layoutstats_res.x
 create mode 100644 dotx.d/type_io_info.x
 create mode 100644 nfsv42_middle_op_layoutstats.xml

diff --git a/Makefile b/Makefile
index 6b65a00..3a81c8f 100755
--- a/Makefile
+++ b/Makefile
@@ -355,6 +355,9 @@ AUTOGEN =	\
 		autogen/layoutget_res.xml \
 		autogen/layoutreturn_args.xml \
 		autogen/layoutreturn_res.xml \
+		autogen/type_io_info.xml \
+		autogen/layoutstats_args.xml \
+		autogen/layoutstats_res.xml \
 		autogen/link_args.xml \
 		autogen/link_res.xml \
 		autogen/lock_args.xml \
@@ -448,6 +451,7 @@ IDXMLSRC_BASE = \
 	${DOC_PREFIX}_middle_op_deallocate.xml \
 	${DOC_PREFIX}_middle_op_io_advise.xml \
 	${DOC_PREFIX}_middle_op_layoutreturn.xml \
+	${DOC_PREFIX}_middle_op_layoutstats.xml \
 	${DOC_PREFIX}_middle_op_offload_cancel.xml \
 	${DOC_PREFIX}_middle_op_offload_status.xml \
 	${DOC_PREFIX}_middle_op_read_plus.xml \
diff --git a/dotx.d/Makefile b/dotx.d/Makefile
index 928bd5b..555b9ee 100755
--- a/dotx.d/Makefile
+++ b/dotx.d/Makefile
@@ -106,16 +106,16 @@ FORE_OPS_AS_IS2 = \
 		want_delegation_res.x \
 		destroy_clientid_args.x \
 		destroy_clientid_res.x \
-                reclaim_complete_args.x \
-                reclaim_complete_res.x \
-                copy_args.x \
-                copy_res.x \
-                copy_notify_args.x \
-                copy_notify_res.x \
-                offload_cancel_args.x \
-                offload_cancel_res.x \
-                offload_status_args.x \
-                offload_status_res.x \
+		reclaim_complete_args.x \
+		reclaim_complete_res.x \
+		copy_args.x \
+		copy_res.x \
+		copy_notify_args.x \
+		copy_notify_res.x \
+		offload_cancel_args.x \
+		offload_cancel_res.x \
+		offload_status_args.x \
+		offload_status_res.x \
 		allocate_args.x \
 		allocate_res.x \
 		deallocate_args.x \
@@ -123,6 +123,9 @@ FORE_OPS_AS_IS2 = \
 		type_io_advise.x \
 		io_advise_args.x \
 		io_advise_res.x \
+		type_io_info.x \
+		layoutstats_args.x \
+		layoutstats_res.x \
 		read_plus_args.x \
 		read_plus_content.x \
 		read_plus_res_pre.x \
diff --git a/dotx.d/layoutstats_args.x b/dotx.d/layoutstats_args.x
new file mode 100644
index 0000000..61e8eee
--- /dev/null
+++ b/dotx.d/layoutstats_args.x
@@ -0,0 +1,10 @@
+
+struct LAYOUTSTATS4args {
+        /* CURRENT_FH: file */
+        offset4                 lsa_offset;
+        length4                 lsa_length;
+        stateid4                lsa_stateid;
+	io_info4		lsa_read;
+	io_info4		lsa_write;
+        layoutupdate4           lsa_layoutupdate;
+};
diff --git a/dotx.d/layoutstats_res.x b/dotx.d/layoutstats_res.x
new file mode 100644
index 0000000..6928863
--- /dev/null
+++ b/dotx.d/layoutstats_res.x
@@ -0,0 +1,5 @@
+
+struct LAYOUTSTATS4res {
+	nfsstat4	lsr_status;
+};
+
diff --git a/dotx.d/type_io_info.x b/dotx.d/type_io_info.x
new file mode 100644
index 0000000..fa1a22a
--- /dev/null
+++ b/dotx.d/type_io_info.x
@@ -0,0 +1,4 @@
+struct io_info4 {
+	uint32_t	ii_count;
+	uint64_t	ii_bytes;
+};
diff --git a/nfsv42_middle_errors.xml b/nfsv42_middle_errors.xml
index c8ecd2d..73f255c 100644
--- a/nfsv42_middle_errors.xml
+++ b/nfsv42_middle_errors.xml
@@ -1955,6 +1955,37 @@
         NFS4ERR_WRONG_TYPE
       </c>
 
+      <c>LAYOUTSTATS</c>
+      <c>
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOTSUPP,
+        NFS4ERR_NO_GRACE,
+        NFS4ERR_OLD_STATEID,
+        NFS4ERR_OP_NOT_IN_SESSION,
+        NFS4ERR_REP_TOO_BIG,
+        NFS4ERR_REP_TOO_BIG_TO_CACHE,
+        NFS4ERR_REQ_TOO_BIG,
+        NFS4ERR_RETRY_UNCACHED_REP,
+        NFS4ERR_SERVERFAULT,
+        NFS4ERR_STALE,
+        NFS4ERR_TOO_MANY_OPS,
+        NFS4ERR_UNKNOWN_LAYOUTTYPE,
+        NFS4ERR_WRONG_CRED,
+        NFS4ERR_WRONG_TYPE
+      </c>
+
       <c>LINK</c>
       <c>
         NFS4ERR_ACCESS,
diff --git a/nfsv42_middle_new_errors.xml b/nfsv42_middle_new_errors.xml
index 053a629..cc18168 100644
--- a/nfsv42_middle_new_errors.xml
+++ b/nfsv42_middle_new_errors.xml
@@ -269,6 +269,37 @@
         NFS4ERR_WRONG_TYPE
       </c>
 
+      <c>LAYOUTSTATS</c>
+      <c>
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOTSUPP,
+        NFS4ERR_NO_GRACE,
+        NFS4ERR_OLD_STATEID,
+        NFS4ERR_OP_NOT_IN_SESSION,
+        NFS4ERR_REP_TOO_BIG,
+        NFS4ERR_REP_TOO_BIG_TO_CACHE,
+        NFS4ERR_REQ_TOO_BIG,
+        NFS4ERR_RETRY_UNCACHED_REP,
+        NFS4ERR_SERVERFAULT,
+        NFS4ERR_STALE,
+        NFS4ERR_TOO_MANY_OPS,
+        NFS4ERR_UNKNOWN_LAYOUTTYPE,
+        NFS4ERR_WRONG_CRED,
+        NFS4ERR_WRONG_TYPE
+      </c>
+
       <c>OFFLOAD_CANCEL</c>
       <c>
         NFS4ERR_ADMIN_REVOKED,
diff --git a/nfsv42_middle_op_layoutstats.xml b/nfsv42_middle_op_layoutstats.xml
new file mode 100644
index 0000000..4784b13
--- /dev/null
+++ b/nfsv42_middle_op_layoutstats.xml
@@ -0,0 +1,38 @@
+<!-- Copyright (C) The IETF Trust (2014) -->
+<!-- Copyright (C) The Internet Society (2014) -->
+
+<section anchor='op:layout_event' title='Operation 70: LAYOUTSTATS - Provide Statistics for the Layout'>
+  <section toc='exclude' title="ARGUMENT">
+    <?rfc include='autogen/type_layoutupdate4.xml'?>
+    <?rfc include='autogen/type_io_info.xml'?>
+    <?rfc include='autogen/layoutstats_args.xml'?>
+  </section>
+
+  <section toc='exclude' title="RESULT">
+    <?rfc include='autogen/layoutstats_res.xml'?>
+  </section>
+
+  <section toc='exclude' title="DESCRIPTION">
+    <t>
+      The client can use LAYOUTSTATS to inform the metadata server
+      about its interaction with the layout represented by the
+      current filehandle, client ID (derived from the session ID
+      in the preceding SEQUENCE operation), byte-range (lea_offset
+      + lea_length), and lea_stateid. lsa_read and lsa_write allow
+      for non-Layout Type specific statistices to be reported.  The
+      remaining information the client is presenting is specific
+      to the Layout Type and presented in the lea_layoutupdate
+      field. Each Layout Type MUST define the contents of
+      lea_layoutupdate in their respective specifications.
+    </t>
+
+    <t>
+      LAYOUTSTATS can be combined with IO_ADVISE (see
+      <xref target='op:io_advise' />) to augment the decision making
+      process of how the metadata server handles a file.  I.e.,
+      IO_ADVISE lets the server know that a byte range has a certain
+      characteristic, but not necessarily the intensity of that
+      characteristic.
+    </t>
+  </section>
+</section>
diff --git a/nfsv42_middle_op_mandlist.xml b/nfsv42_middle_op_mandlist.xml
index 4ebd0bb..b590a6e 100644
--- a/nfsv42_middle_op_mandlist.xml
+++ b/nfsv42_middle_op_mandlist.xml
@@ -141,6 +141,7 @@
       <c>LAYOUTCOMMIT         </c> <c> OPT </c> <c> pNFS (REQ)   </c>
       <c>LAYOUTGET            </c> <c> OPT </c> <c> pNFS (REQ)   </c>
       <c>LAYOUTRETURN         </c> <c> OPT </c> <c> pNFS (REQ)   </c>
+      <c>LAYOUTSTATS          </c> <c> OPT </c> <c> pNFS (REQ)   </c>
       <c>LINK                 </c> <c> OPT </c> <c>              </c>
       <c>LOCK                 </c> <c> REQ </c> <c>              </c>
       <c>LOCKT                </c> <c> REQ </c> <c>              </c>
-- 
1.9.0

---

From: Thomas Haynes <[email-masked]>
Date: Thu, 15 May 2014 17:56:09 -0700

Signed-off-by: Tom Haynes <[email-masked]>
---
 dotx.d/spit_types.sh                | 19 +++++++++++--------
 nfsv42_middle_op_layouterror.xml    | 11 ++++++++++-
 nfsv42_middle_op_layoutstats.xml    |  8 +++++++-
 nfsv42_middle_op_offload_cancel.xml |  2 +-
 nfsv42_middle_op_offload_status.xml |  2 +-
 nfsv42_middle_op_read_plus.xml      |  2 +-
 nfsv42_middle_op_seek.xml           |  2 +-
 nfsv42_middle_op_write_same.xml     |  2 +-
 8 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/dotx.d/spit_types.sh b/dotx.d/spit_types.sh
index 980ad23..bb1dadc 100755
--- a/dotx.d/spit_types.sh
+++ b/dotx.d/spit_types.sh
@@ -983,12 +983,13 @@ enum nfs_opnum4 {
  OP_COPY_NOTIFY		= 61,
  OP_DEALLOCATE		= 62,
  OP_IO_ADVISE		= 63,
- OP_XXX_FOR_NOW		= 64,
- OP_OFFLOAD_CANCEL	= 65,
- OP_OFFLOAD_STATUS	= 66,
- OP_READ_PLUS		= 67,
- OP_SEEK		= 68,
- OP_WRITE_SAME		= 69,
+ OP_LAYOUTERROR		= 64,
+ OP_LAYOUTSTATS		= 65,
+ OP_OFFLOAD_CANCEL	= 66,
+ OP_OFFLOAD_STATUS	= 67,
+ OP_READ_PLUS		= 68,
+ OP_SEEK		= 69,
+ OP_WRITE_SAME		= 70,
  OP_ILLEGAL		= 10044
 };
 EOF
@@ -1105,7 +1106,8 @@ union nfs_argop4 switch (nfs_opnum4 argop) {
  case OP_COPY_NOTIFY:	COPY_NOTIFY4args opoffload_notify;
  case OP_DEALLOCATE:	DEALLOCATE4args opdeallocate;
  case OP_IO_ADVISE:	IO_ADVISE4args opio_advise;
- case OP_XXX_FOR_NOW:	void;
+ case OP_LAYOUTERROR: LAYOUTERRORargs4  oplayouterror;
+ case OP_LAYOUTSTATS: LAYOUTSTATSargs4  oplayoutstats;
  case OP_OFFLOAD_CANCEL: OFFLOAD_CANCEL4args opoffload_cancel;
  case OP_OFFLOAD_STATUS: OFFLOAD_STATUS4args opoffload_status;
  case OP_READ_PLUS:	READ_PLUS4args opread_plus;
@@ -1237,7 +1239,8 @@ union nfs_resop4 switch (nfs_opnum4 resop) {
  case OP_COPY_NOTIFY:	COPY_NOTIFY4res opcopy_notify;
  case OP_DEALLOCATE:	DEALLOCATE4res opdeallocate;
  case OP_IO_ADVISE:	IO_ADVISE4res opio_advise;
- case OP_XXX_FOR_NOW:	void;
+ case OP_LAYOUTERROR: LAYOUTERROR4res oplayouterror;
+ case OP_LAYOUTSTATS: LAYOUTSTATS4res oplayputstats;
  case OP_OFFLOAD_CANCEL: OFFLOAD_CANCEL4res opoffload_cancel;
  case OP_OFFLOAD_STATUS: OFFLOAD_STATUS4res opoffload_status;
  case OP_READ_PLUS:	READ_PLUS4res opread_plus;
diff --git a/nfsv42_middle_op_layouterror.xml b/nfsv42_middle_op_layouterror.xml
index e51d46d..1b51410 100644
--- a/nfsv42_middle_op_layouterror.xml
+++ b/nfsv42_middle_op_layouterror.xml
@@ -1,7 +1,7 @@
 <!-- Copyright (C) The IETF Trust (2011-2014) -->
 <!-- Copyright (C) The Internet Society (2011-2014) -->
 
-<section anchor='op:layouterror' title='Operation XX: LAYOUTERROR - Provide Errors for the Layout'>
+<section anchor='op:layouterror' title='Operation 64: LAYOUTERROR - Provide Errors for the Layout'>
   <section toc='exclude' title="ARGUMENT">
     <?rfc include='autogen/type_layoutupdate4.xml'?>
     <?rfc include='autogen/type_device_error.xml'?>
@@ -44,6 +44,15 @@
         </t>
       </list>
     </t>
+
+    <t>
+      Note that while the metadata server may return an error
+      associated with the layout stateid or the open file, it
+      MUST NOT return an error in the processing of the errors.
+      If LAYOUTERROR is in a compound before LAYOUTRETURN, it
+      MUST NOT introduce an error other than what LAYOUTRETURN
+      would already encounter.
+    </t>
   </section>
 
   <section anchor="sec:LAYOUTERROR:impl" toc='exclude' title="IMPLEMENTATION">
diff --git a/nfsv42_middle_op_layoutstats.xml b/nfsv42_middle_op_layoutstats.xml
index 93f3747..42da360 100644
--- a/nfsv42_middle_op_layoutstats.xml
+++ b/nfsv42_middle_op_layoutstats.xml
@@ -1,7 +1,7 @@
 <!-- Copyright (C) The IETF Trust (2014) -->
 <!-- Copyright (C) The Internet Society (2014) -->
 
-<section anchor='op:layoutstats' title='Operation 70: LAYOUTSTATS - Provide Statistics for the Layout'>
+<section anchor='op:layoutstats' title='Operation 65: LAYOUTSTATS - Provide Statistics for the Layout'>
   <section toc='exclude' title="ARGUMENT">
     <?rfc include='autogen/type_layoutupdate4.xml'?>
     <?rfc include='autogen/type_io_info.xml'?>
@@ -34,5 +34,11 @@
       characteristic, but not necessarily the intensity of that
       characteristic.
     </t>
+
+    <t>
+      Note that while the metadata server may return an error
+      associated with the layout stateid or the open file, it
+      MUST NOT return an error in the processing of the statistics.
+    </t>
   </section>
 </section>
diff --git a/nfsv42_middle_op_offload_cancel.xml b/nfsv42_middle_op_offload_cancel.xml
index 9b1a9d8..fd85ec7 100644
--- a/nfsv42_middle_op_offload_cancel.xml
+++ b/nfsv42_middle_op_offload_cancel.xml
@@ -1,7 +1,7 @@
 <!-- Copyright (C) The IETF Trust (2011-2014) -->
 <!-- Copyright (C) The Internet Society (2011-2014) -->
 
-<section anchor="op:offload_close" title="Operation 65: OFFLOAD_CANCEL - Stop an Offloaded Operation">
+<section anchor="op:offload_close" title="Operation 66: OFFLOAD_CANCEL - Stop an Offloaded Operation">
   <section toc='exclude' title="ARGUMENT">
     <?rfc include='autogen/offload_cancel_args.xml'?>
   </section>
diff --git a/nfsv42_middle_op_offload_status.xml b/nfsv42_middle_op_offload_status.xml
index d7f6aee..0d7a150 100644
--- a/nfsv42_middle_op_offload_status.xml
+++ b/nfsv42_middle_op_offload_status.xml
@@ -1,7 +1,7 @@
 <!-- Copyright (C) The IETF Trust (2011-2014) -->
 <!-- Copyright (C) The Internet Society (2011-2014) -->
 
-<section anchor="op:offload_status" title="Operation 66: OFFLOAD_STATUS - Poll for Status of Asynchronous Operation">
+<section anchor="op:offload_status" title="Operation 67: OFFLOAD_STATUS - Poll for Status of Asynchronous Operation">
   <section toc='exclude' title="ARGUMENT">
     <?rfc include='autogen/offload_status_args.xml'?>
   </section>
diff --git a/nfsv42_middle_op_read_plus.xml b/nfsv42_middle_op_read_plus.xml
index 9dda772..d29350b 100644
--- a/nfsv42_middle_op_read_plus.xml
+++ b/nfsv42_middle_op_read_plus.xml
@@ -1,7 +1,7 @@
 <!-- Copyright (C) The IETF Trust (2011-2014) -->
 <!-- Copyright (C) The Internet Society (2011-2014) -->
 
-<section anchor='op:read_plus' title='Operation 67: READ_PLUS - READ Data or Holes from a File'>
+<section anchor='op:read_plus' title='Operation 68: READ_PLUS - READ Data or Holes from a File'>
   <section toc='exclude' title="ARGUMENT">
     <?rfc include='autogen/read_plus_args.xml'?>
   </section>
diff --git a/nfsv42_middle_op_seek.xml b/nfsv42_middle_op_seek.xml
index d21c361..8679abf 100644
--- a/nfsv42_middle_op_seek.xml
+++ b/nfsv42_middle_op_seek.xml
@@ -1,7 +1,7 @@
 <!-- Copyright (C) The IETF Trust (2011-2014) -->
 <!-- Copyright (C) The Internet Society (2011-2014) -->
 
-<section anchor='op:seek' title='Operation 68: SEEK - Find the Next Data or Hole'>
+<section anchor='op:seek' title='Operation 69: SEEK - Find the Next Data or Hole'>
   <section toc='exclude' title="ARGUMENT">
     <?rfc include='autogen/seek_args.xml'?>
   </section>
diff --git a/nfsv42_middle_op_write_same.xml b/nfsv42_middle_op_write_same.xml
index 5845c33..4a582f8 100644
--- a/nfsv42_middle_op_write_same.xml
+++ b/nfsv42_middle_op_write_same.xml
@@ -1,7 +1,7 @@
 <!-- Copyright (C) The IETF Trust (2011-2014) -->
 <!-- Copyright (C) The Internet Society (2011-2014) -->
 
-<section anchor='op:write_same' title='Operation 69: WRITE_SAME - WRITE an ADB Multiple Times to a File'>
+<section anchor='op:write_same' title='Operation 70: WRITE_SAME - WRITE an ADB Multiple Times to a File'>
   <section toc='exclude' title="ARGUMENT">
     <?rfc include='autogen/stable_how4.xml'?>
     <?rfc include='autogen/app_data_block4.xml'?>
-- 
1.9.0

---

From: Thomas Haynes <[email-masked]>
Date: Thu, 15 May 2014 17:56:08 -0700

Signed-off-by: Tom Haynes <[email-masked]>
---
 Makefile                          |   7 +-
 dotx.d/Makefile                   |   6 +-
 dotx.d/fore_prog.x                |   6 --
 dotx.d/layout_errors.x            |  14 ---
 dotx.d/layouterror_args.x         |   8 ++
 dotx.d/layouterror_res.x          |   5 ++
 dotx.d/spit_types.sh              |  15 ----
 dotx.d/type_device_error.x        |   7 ++
 nfsv42_middle_errors.xml          |  31 +++++++
 nfsv42_middle_new_errors.xml      |  31 +++++++
 nfsv42_middle_op_layouterror.xml  | 147 ++++++++++++++++++++++++++++++
 nfsv42_middle_op_layoutreturn.xml | 185 --------------------------------------
 nfsv42_middle_op_layoutstats.xml  |   2 +-
 nfsv42_middle_op_mandlist.xml     |   3 +-
 14 files changed, 240 insertions(+), 227 deletions(-)
 delete mode 100644 dotx.d/layout_errors.x
 create mode 100644 dotx.d/layouterror_args.x
 create mode 100644 dotx.d/layouterror_res.x
 create mode 100644 dotx.d/type_device_error.x
 create mode 100644 nfsv42_middle_op_layouterror.xml
 delete mode 100644 nfsv42_middle_op_layoutreturn.xml

diff --git a/Makefile b/Makefile
index 3a81c8f..669d0dd 100755
--- a/Makefile
+++ b/Makefile
@@ -125,7 +125,6 @@ SPITGEN =	dotx.d/type_nfstime4.x \
 		dotx.d/type_ssv_seal_plain_tkn4.x \
 		dotx.d/type_ssv_seal_cipher_tkn4.x \
 		dotx.d/type_layoutreturn4.x \
-		dotx.d/type_layoutreturn_errs.x \
 		dotx.d/type_client_owner4.x \
 		dotx.d/type_server_owner4.x \
 		dotx.d/type_device_addr4.x \
@@ -203,7 +202,6 @@ SPITGENXML =	autogen/type_nfstime4.xml \
 		autogen/type_ssv_seal_plain_tkn4.xml \
 		autogen/type_ssv_seal_cipher_tkn4.xml \
 		autogen/type_layoutreturn4.xml \
-		autogen/type_layoutreturn_errs.xml \
 		autogen/type_client_owner4.xml \
 		autogen/type_server_owner4.xml \
 		autogen/type_device_addr4.xml \
@@ -351,6 +349,9 @@ AUTOGEN =	\
 		autogen/io_advise_res.xml \
 		autogen/layoutcommit_args.xml \
 		autogen/layoutcommit_res.xml \
+		autogen/type_device_error.xml \
+		autogen/layouterror_args.xml \
+		autogen/layouterror_res.xml \
 		autogen/layoutget_args.xml \
 		autogen/layoutget_res.xml \
 		autogen/layoutreturn_args.xml \
@@ -450,7 +451,7 @@ IDXMLSRC_BASE = \
 	${DOC_PREFIX}_middle_op_exchange_id.xml \
 	${DOC_PREFIX}_middle_op_deallocate.xml \
 	${DOC_PREFIX}_middle_op_io_advise.xml \
-	${DOC_PREFIX}_middle_op_layoutreturn.xml \
+	${DOC_PREFIX}_middle_op_layouterror.xml \
 	${DOC_PREFIX}_middle_op_layoutstats.xml \
 	${DOC_PREFIX}_middle_op_offload_cancel.xml \
 	${DOC_PREFIX}_middle_op_offload_status.xml \
diff --git a/dotx.d/Makefile b/dotx.d/Makefile
index 555b9ee..d2e81c3 100755
--- a/dotx.d/Makefile
+++ b/dotx.d/Makefile
@@ -123,6 +123,9 @@ FORE_OPS_AS_IS2 = \
 		type_io_advise.x \
 		io_advise_args.x \
 		io_advise_res.x \
+		type_device_error.x \
+		layouterror_args.x \
+		layouterror_res.x \
 		type_io_info.x \
 		layoutstats_args.x \
 		layoutstats_res.x \
@@ -196,7 +199,6 @@ SPITGEN =	basic_types.x \
 		type_ssv_seal_plain_tkn4.x \
 		type_ssv_seal_cipher_tkn4.x \
 		type_layoutreturn4.x \
-		type_layoutreturn_errs.x \
 		type_client_owner4.x \
 		type_server_owner4.x \
 		type_device_addr4.x \
@@ -255,7 +257,7 @@ AUTOGEN =	hand_edit_warning.x \
 
 all:	nfsv42.x filelist
 
-nfsv42.x: $(AUTOGEN) hand_edit_warning.x head.x fs_locations_info.x file_layout.x fore_ops.x layout_errors.x fore_prog.x \
+nfsv42.x: $(AUTOGEN) hand_edit_warning.x head.x fs_locations_info.x file_layout.x fore_ops.x fore_prog.x \
 		back_ops.x back_prog.x
 	m4 hand_edit_warning.x head.x fs_locations_info.x file_layout.x fore_ops.x fore_prog.x back_ops.x back_prog.x | expand | tr -d "\r" > nfsv42.tmp
 	mv nfsv42.tmp nfsv42.x
diff --git a/dotx.d/fore_prog.x b/dotx.d/fore_prog.x
index 5eb336e..d0b92c6 100644
--- a/dotx.d/fore_prog.x
+++ b/dotx.d/fore_prog.x
@@ -10,12 +10,6 @@ include(type_COMPOUND4args.x)
 include(type_COMPOUND4res.x)
 
 /*
- * Layout return errors, which might
- * include the nfs_opnum4.
- */
-include(layout_errors.x)
-
-/*
  * Remote file service routines
  */
 program NFS4_PROGRAM {
diff --git a/dotx.d/layout_errors.x b/dotx.d/layout_errors.x
deleted file mode 100644
index 2ed5df8..0000000
--- a/dotx.d/layout_errors.x
+++ /dev/null
@@ -1,14 +0,0 @@
-
-%/*
-% * Encoded in the lou_body field of data type layoutupdate4:
-% *	 Nothing. lou_body is a zero length array of bytes.
-% */
-%
-
-%/*
-% * Encoded in the lrf_body field of
-% * data type layoutreturn_file4:
-% */
-include(type_layoutreturn_errs.x)
-%
-
diff --git a/dotx.d/layouterror_args.x b/dotx.d/layouterror_args.x
new file mode 100644
index 0000000..8d1fcaf
--- /dev/null
+++ b/dotx.d/layouterror_args.x
@@ -0,0 +1,8 @@
+
+struct LAYOUTERROR4args {
+        /* CURRENT_FH: file */
+        offset4			lea_offset;
+        length4			lea_length;
+        stateid4		lea_stateid;
+        device_error4		lea_errors;
+};
diff --git a/dotx.d/layouterror_res.x b/dotx.d/layouterror_res.x
new file mode 100644
index 0000000..921348a
--- /dev/null
+++ b/dotx.d/layouterror_res.x
@@ -0,0 +1,5 @@
+
+struct LAYOUTERROR4res {
+	nfsstat4	ler_status;
+};
+
diff --git a/dotx.d/spit_types.sh b/dotx.d/spit_types.sh
index 4a6d254..980ad23 100755
--- a/dotx.d/spit_types.sh
+++ b/dotx.d/spit_types.sh
@@ -470,21 +470,6 @@ union layoutreturn4 switch(layoutreturn_type4 lr_returntype) {
 EOF
         ;;
 
-	type_layoutreturn_errs.x )
-
-cat << EOF > $i
-struct layoutreturn_device_error4 {
-	deviceid4	lrde_deviceid;
-	nfsstat4	lrde_status;
-	nfs_opnum4	lrde_opnum;
-};
-
-struct layoutreturn_error_report4 {
-	layoutreturn_device_error4	lrer_errors<>;
-};
-EOF
-        ;;
-
 	type_client_owner4.x )
 
 cat << EOF > $i
diff --git a/dotx.d/type_device_error.x b/dotx.d/type_device_error.x
new file mode 100644
index 0000000..8fc6733
--- /dev/null
+++ b/dotx.d/type_device_error.x
@@ -0,0 +1,7 @@
+
+struct device_error4 {
+	deviceid4	de_deviceid;
+	nfsstat4	de_status;
+	nfs_opnum4	de_opnum;
+};
+
diff --git a/nfsv42_middle_errors.xml b/nfsv42_middle_errors.xml
index 73f255c..ef7a5f9 100644
--- a/nfsv42_middle_errors.xml
+++ b/nfsv42_middle_errors.xml
@@ -1885,6 +1885,37 @@
         NFS4ERR_WRONG_CRED
       </c>
 
+      <c>LAYOUTERROR</c>
+      <c>
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOTSUPP,
+        NFS4ERR_NO_GRACE,
+        NFS4ERR_OLD_STATEID,
       <c>LAYOUTGET</c>
index cc18168..17d31a4 100644
 
+      <c>LAYOUTERROR</c>
+      <c>
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOTSUPP,
+        NFS4ERR_NO_GRACE,
+        NFS4ERR_OLD_STATEID,
+        NFS4ERR_OP_NOT_IN_SESSION,
+        NFS4ERR_REP_TOO_BIG,
+        NFS4ERR_REP_TOO_BIG_TO_CACHE,
+        NFS4ERR_REQ_TOO_BIG,
+        NFS4ERR_RETRY_UNCACHED_REP,
+        NFS4ERR_SERVERFAULT,
+        NFS4ERR_STALE,
+        NFS4ERR_TOO_MANY_OPS,
+        NFS4ERR_UNKNOWN_LAYOUTTYPE,
+        NFS4ERR_WRONG_CRED,
+        NFS4ERR_WRONG_TYPE
+      </c>
+
       <c>LAYOUTSTATS</c>
       <c>
         NFS4ERR_ADMIN_REVOKED,
diff --git a/nfsv42_middle_op_layouterror.xml b/nfsv42_middle_op_layouterror.xml
new file mode 100644
index 0000000..e51d46d
--- /dev/null
+++ b/nfsv42_middle_op_layouterror.xml
@@ -0,0 +1,147 @@
+<!-- Copyright (C) The IETF Trust (2011-2014) -->
+<!-- Copyright (C) The Internet Society (2011-2014) -->
+
+<section anchor='op:layouterror' title='Operation XX: LAYOUTERROR - Provide Errors for the Layout'>
+  <section toc='exclude' title="ARGUMENT">
+    <?rfc include='autogen/type_layoutupdate4.xml'?>
+    <?rfc include='autogen/type_device_error.xml'?>
+    <?rfc include='autogen/layouterror_args.xml'?>
+  </section>
+
+  <section toc='exclude' title="RESULT">
+    <?rfc include='autogen/layouterror_res.xml'?>
+  </section>
+
+  <section toc='exclude' title="DESCRIPTION">
+    <t>
+      The client can use LAYOUTERROR to inform the metadata server
+      about errors in its interaction with the layout represented by the
+      current filehandle, client ID (derived from the session ID
+      in the preceding SEQUENCE operation), byte-range (lea_offset
+      + lea_length), and lea_stateid.
+    </t>
+
+    <t>
+      Each individual device_error4 describes a single error
+      associated with a storage device, which is identified via
+      de_deviceid.  The operation which returned the error is
+      identified via de_opnum. Finally the NFS error value (nfsstat4)
+      encountered is provided via de_status and may consist of the
+      following error codes:
+    </t>
+
+    <t>
+      <list style='hanging'>
+        <t hangText='NFS4ERR_NXIO:'>
+          The client was unable to establish any communication with
+          the storage device.
+        </t>
+
+        <t hangText='NFS4ERR_*:'>
+          The client was able to establish communication with the
+          storage device and is returning one of the allowed error
+          codes for the operation denoted by de_opnum.
+        </t>
+      </list>
+    </t>
+  </section>
+
+  <section anchor="sec:LAYOUTERROR:impl" toc='exclude' title="IMPLEMENTATION">
+    <t>
+      There are two broad classes of errors, transient and persistent.
+      The client SHOULD strive to only use this new mechanism to
+      report persistent errors. It MUST be able to deal with transient
+      issues by itself. Also, while the client might consider an
+      issue to be persistent, it MUST be prepared for the metadata
+      server to consider such issues to be transient. A prime example
+      of this is if the metadata server fences off a client from
+      either a stateid or a filehandle.  The client will get an
+      error from the storage device and might relay either
+      NFS4ERR_ACCESS or NFS4ERR_BAD_STATEID back to the metadata
+      server, with the belief that this is a hard error. If the
+      metadata server is informed by the client that there is an
+      error, it can safely ignore that.  For it, the mission is
+      accomplished in that the client has returned a layout that
+      the metadata server had most likely recalled.
+    </t>
+
+    <t>
+      The client might also need to inform the metadata server that
+      it cannot reach one or more of the storage devices. While the
+      metadata server can detect the connectivity of both of these
+      paths:
+
+      <list style='symbols'>
+        <t>
+          metadata server to storage device
+        </t>
+
+        <t>
+          metadata server to client
+        </t>
+      </list>
+
+      it cannot determine if the client and storage device path is
+      working. As with the case of the storage device passing errors
+      to the client, it must be prepared for the metadata server
+      to consider such outages as being transitory.
+    </t>
+
+    <t>
+      Clients are expected to tolerate transient storage device
+      errors, and hence clients SHOULD NOT use the LAYOUTERROR
+      error handling for device access problems that may be transient.
+      The methods by which a client decides whether a device access
+      problem is transient vs persistent are implementation-specific,
+      but may include retrying I/Os to a data server under appropriate
+      conditions.
+    </t>
+
+    <t>
+      When an I/O fails to a storage device, the client SHOULD retry
+      the failed I/O via the metadata server. In this situation,
+      before retrying the I/O, the client SHOULD return the layout, or
+      the affected portion thereof, and SHOULD indicate which storage
+      device or devices was problematic. The client needs to do this
+      when the storage device is being unresponsive in order to fence
+      off any failed write attempts, and ensure that they do not end
+      up overwriting any later data being written through the metadata
+      server.  If the client does not do this, the metadata server MAY
+      issue a layout recall callback in order to perform the retried I/O.
+    </t>
+
+    <t>
+      The client needs to be cognizant that since this error handling
+      is optional in the metadata server, the metadata server may
+      silently ignore this functionality.  Also, as the metadata
+      server may consider some issues the client reports to be
+      expected, the client might find it difficult to detect a
+      metadata server which has not implemented error handling via
+      LAYOUTERROR.
+    </t>
+
+    <t>
+      If an metadata server is aware that a storage device is proving
+      problematic to a client, the metadata server SHOULD NOT include
+      that storage device in any pNFS layouts sent to that client. If
+      the metadata server is aware that a storage device is affecting
+      many clients, then the metadata server SHOULD NOT include that
+      storage device in any pNFS layouts sent out.  If a client asks
+      for a new layout for the file from the metadata server, it MUST
+      be prepared for the metadata server to return that storage device
+      in the layout. The metadata server might not have any choice in
+      using the storage device, i.e., there might only be one possible
+      layout for the system.  Also, in the case of existing files, the
+      metadata server might have no choice in which storage devices to
+      hand out to clients.
+    </t>
+
+    <t>
+      The metadata server is not required to indefinitely retain
+      per-client storage device error information. An metadata server is
+      also not required to automatically reinstate use of a previously
+      problematic storage device; administrative intervention may be
+      required instead.
+    </t>
+  </section>
+</section>
diff --git a/nfsv42_middle_op_layoutreturn.xml b/nfsv42_middle_op_layoutreturn.xml
deleted file mode 100644
index ad2f33d..0000000
--- a/nfsv42_middle_op_layoutreturn.xml
+++ /dev/null
@@ -1,185 +0,0 @@
-<!-- Copyright (C) The IETF Trust (2011-2014) -->
-<!-- Copyright (C) The Internet Society (2011-2014) -->
-
-<section anchor="sec:pLeh:LAYOUTRETURN" title="Changes to Operation 51: LAYOUTRETURN">
-  <section anchor="sec:pLeh:intro" toc='exclude' title="Introduction">
-    <t>
-      In the pNFS description provided in <xref target="RFC5661" />, the
-      client is not capable of relaying an error code from the storage
-      device to the metadata server. In the specification of the Object
-      Layout Type <xref target='RFC5664' />, use is made of the opaque
-      lrf_body field of the LAYOUTRETURN argument to do such a relaying
-      of error codes. In this section, we define a new data structure
-      to enable the passing of error codes back to the metadata server
-      and provide some guidelines on what both the client and metadata
-      server should expect in such circumstances.
-    </t>
-
-    <t>
-      There are two broad classes of errors, transient and persistent.
-      The client SHOULD strive to only use this new mechanism to report
-      persistent errors. It MUST be able to deal with transient issues
-      by itself. Also, while the client might consider an issue to
-      be persistent, it MUST be prepared for the metadata server to
-      consider such issues to be transient. A prime example of this is
-      if the metadata server fences off a client from either a stateid
-      or a filehandle.  The client will get an error from the storage
-      device and might relay either NFS4ERR_ACCESS or NFS4ERR_BAD_STATEID
-      back to the metadata server, with the belief that this is a hard
-      error. If the metadata server is informed by the client that there
-      is an error, it can safely ignore that.  For it, the mission is
-      accomplished in that the client has returned a layout that the
-      metadata server had most likely recalled.
-    </t>
-
-    <t>
-      The client might also need to inform the metadata server that
-      it cannot reach one or more of the storage devices. While the
-      metadata server can detect the connectivity of both of these paths:
-
-      <list style='symbols'>
-        <t>
-          metadata server to storage device
-        </t>
-
-        <t>
-          metadata server to client
-        </t>
-      </list>
-
-      it cannot determine if the client and storage device path is
-      working. As with the case of the storage device passing errors to
-      the client, it must be prepared for the metadata server to consider
-      such outages as being transitory.
-    </t>
-
-    <t>
-      The existing LAYOUTRETURN operation is extended by introducing a
-      new data structure to report errors, layoutreturn_device_error4.
-      Also, layoutreturn_device_error4 is introduced to enable an
-      array of errors to be reported.
-    </t>
-  </section>
-
-  <section anchor="sec:pLeh:LAYOUTRETURN:args" toc='exclude' title="ARGUMENT">
-    <t>
-      The ARGUMENT specification of the LAYOUTRETURN operation in section
-      18.44.1 of <xref target="RFC5661" /> is augmented by the following
-      XDR code <xref target="RFC4506" />:
-    </t>
-
-    <?rfc include='autogen/type_layoutreturn_errs.xml'?>
-  </section>
-
-  <section anchor="sec:pLeh:LAYOUTRETURN:res" toc='exclude' title="RESULT">
-    <t>
-      The RESULT of the LAYOUTRETURN operation is unchanged; see section
-      18.44.2 of <xref target="RFC5661" />.
-    </t>
-  </section>
-
-  <section anchor="sec:pLeh:LAYOUTRETURN:desc" toc='exclude' title="DESCRIPTION">
-    <t>
-      The following text is added to the end of the LAYOUTRETURN operation
-      DESCRIPTION in section 18.44.3 of <xref target="RFC5661" />.
-    </t>
-
-    <t>
-      When a client uses LAYOUTRETURN with a type of LAYOUTRETURN4_FILE,
-      then if the lrf_body field is NULL, it indicates to the metadata
-      server that the client experienced no errors. If lrf_body is
-      non-NULL, then the field references error information which is
-      layout type specific. I.e., the Object Layout Type can continue to
-      utilize lrf_body as specified in <xref target='RFC5664' />.  For
-      both File <xref target="RFC5661" /> and Block <xref target="RFC5663"
-      /> Layout Types, the field references a layoutreturn_device_error4,
-      which contains an array of layoutreturn_device_error4.
-    </t>
-
-    <t>
-      Each individual layoutreturn_device_error4 describes a single
-      error associated with a storage device, which is identified
-      via lrde_deviceid.  The operation which returned the error is
-      identified via lrde_opnum. Finally the NFS error value (nfsstat4)
-      encountered is provided via lrde_status and may consist of the
-      following error codes:
-    </t>
-
-    <t>
-      <list style='hanging'>
-        <t hangText='NFS4ERR_NXIO:'>
-          The client was unable to establish any communication with the storage device.
-        </t>
-
-        <t hangText='NFS4ERR_*:'>
-          The client was able to establish communication with the storage device and
-          is returning one of the allowed error codes for the operation
-          denoted by lrde_opnum.
-        </t>
-      </list>
-    </t>
-  </section>
-
-  <section anchor="sec:pLeh:LAYOUTRETURN:impl" toc='exclude' title="IMPLEMENTATION">
-    <t>
-      The following text is added to the end of the LAYOUTRETURN operation
-      IMPLEMENTATION in section 18.4.4 of <xref target="RFC5661" />.
-    </t>
-
-    <t>
-      Clients are expected to tolerate transient storage device errors,
-      and hence clients SHOULD NOT use the LAYOUTRETURN error handling
-      for device access problems that may be transient. The methods by
-      which a client decides whether a device access problem is transient
-      vs persistent are implementation-specific, but may include retrying
-      I/Os to a data server under appropriate conditions.
-    </t>
-
-    <t>
-      When an I/O fails to a storage device, the client SHOULD retry
-      the failed I/O via the metadata server. In this situation,
-      before retrying the I/O, the client SHOULD return the layout, or
-      the affected portion thereof, and SHOULD indicate which storage
-      device or devices was problematic. The client needs to do this
-      when the storage device is being unresponsive in order to fence
-      off any failed write attempts, and ensure that they do not end
-      up overwriting any later data being written through the metadata
-      server.  If the client does not do this, the metadata server MAY
-      issue a layout recall callback in order to perform the retried I/O.
-    </t>
-
-    <t>
-      The client needs to be cognizant that since this error handling is
-      optional in the metadata server, the metadata server may silently
-      ignore this functionality.  Also, as the metadata server may
-      consider some issues the client reports to be expected (see <xref
-      target='sec:pLeh:intro' />), the client might find it difficult to
-      detect a metadata server which has not implemented error handling
-      via LAYOUTRETURN.
-    </t>
-
-    <t>
-      If an metadata server is aware that a storage device is proving
-      problematic to a client, the metadata server SHOULD NOT include
-      that storage device in any pNFS layouts sent to that client. If
-      the metadata server is aware that a storage device is affecting
-      many clients, then the metadata server SHOULD NOT include that
-      storage device in any pNFS layouts sent out.  If a client asks
-      for a new layout for the file from the metadata server, it MUST
-      be prepared for the metadata server to return that storage device
-      in the layout. The metadata server might not have any choice in
-      using the storage device, i.e., there might only be one possible
-      layout for the system.  Also, in the case of existing files, the
-      metadata server might have no choice in which storage devices to
-      hand out to clients.
-    </t>
-
-    <t>
-      The metadata server is not required to indefinitely retain
-      per-client storage device error information. An metadata server is
-      also not required to automatically reinstate use of a previously
-      problematic storage device; administrative intervention may be
-      required instead.
-    </t>
-  </section>
-</section>
diff --git a/nfsv42_middle_op_layoutstats.xml b/nfsv42_middle_op_layoutstats.xml
index 4784b13..93f3747 100644
--- a/nfsv42_middle_op_layoutstats.xml
+++ b/nfsv42_middle_op_layoutstats.xml
@@ -1,7 +1,7 @@
 <!-- Copyright (C) The IETF Trust (2014) -->
 <!-- Copyright (C) The Internet Society (2014) -->
 
-<section anchor='op:layout_event' title='Operation 70: LAYOUTSTATS - Provide Statistics for the Layout'>
+<section anchor='op:layoutstats' title='Operation 70: LAYOUTSTATS - Provide Statistics for the Layout'>
   <section toc='exclude' title="ARGUMENT">
     <?rfc include='autogen/type_layoutupdate4.xml'?>
     <?rfc include='autogen/type_io_info.xml'?>
diff --git a/nfsv42_middle_op_mandlist.xml b/nfsv42_middle_op_mandlist.xml
index b590a6e..8664f06 100644
--- a/nfsv42_middle_op_mandlist.xml
+++ b/nfsv42_middle_op_mandlist.xml
@@ -141,7 +141,8 @@
       <c>LAYOUTCOMMIT         </c> <c> OPT </c> <c> pNFS (REQ)   </c>
       <c>LAYOUTGET            </c> <c> OPT </c> <c> pNFS (REQ)   </c>
       <c>LAYOUTRETURN         </c> <c> OPT </c> <c> pNFS (REQ)   </c>
-      <c>LAYOUTSTATS          </c> <c> OPT </c> <c> pNFS (REQ)   </c>
+      <c>LAYOUTERROR          </c> <c> OPT </c> <c> pNFS (OPT)   </c>
+      <c>LAYOUTSTATS          </c> <c> OPT </c> <c> pNFS (OPT)   </c>
       <c>LINK                 </c> <c> OPT </c> <c>              </c>
       <c>LOCK                 </c> <c> REQ </c> <c>              </c>
       <c>LOCKT                </c> <c> REQ </c> <c>              </c>
-- 
1.9.0

---

From: Christoph Hellwig <[email-masked]>
Date: Fri, 16 May 2014 10:20:39 +0200

Is it intentional that this uses the same layoutupdate4 as LAYOUTCOMMIT
does in NFSv4.2?  To me this would imply the layout-specific blob is the
same as passed to LAYOUTCOMMIT.

---

From: Christoph Hellwig <[email-masked]>
Date: Fri, 16 May 2014 10:23:28 +0200

> +      encountered is provided via de_status and may consist of the

What would the client put into de_opnum for block or object layouts?

---

From: Tom Haynes <[email-masked]>
Date: Fri, 16 May 2014 10:24:31 -0500

On May 16, 2014, at 3:20 AM, Christoph Hellwig <[email-masked]> wrote:

> does in NFSv4.2?  

Yes

> To me this would imply the layout-specific blob is the

No, it can be defined differently.

I don’t see the need for a new data structure to hold this info.

---

From: Tom Haynes <[email-masked]>
Date: Fri, 16 May 2014 10:43:15 -0500

> What would the client put into de_opnum for block or object layouts?

The allowable storage device ops are the data-server housekeeping
and data-server I/O operations. A Layout Type like Flex Files can
report on the housekeeping ones. For the Block or Object Layout Types,
those ops would not make sense.

So everything else comes back to COMMIT, READ, WRITE, and PUTFH.

Are there any other ops used for block or object layout drivers?

And the spec could define that if the de_opnum is ILLEGAL, it
signifies that it was a driver specific operation.

The other question is whether the client is mapping errors to nfsstat4 or
using something like:

   /// enum pnfs_osd_errno4 {
   ///     PNFS_OSD_ERR_EIO            = 1,
   ///     PNFS_OSD_ERR_NOT_FOUND      = 2,
   ///     PNFS_OSD_ERR_NO_SPACE       = 3,
   ///     PNFS_OSD_ERR_BAD_CRED       = 4,
   ///     PNFS_OSD_ERR_NO_ACCESS      = 5,
   ///     PNFS_OSD_ERR_UNREACHABLE    = 6,
   ///     PNFS_OSD_ERR_RESOURCE       = 7
   /// };
I think the client should do the mapping.

---

From: Christoph Hellwig <[email-masked]>
Date: Sat, 17 May 2014 11:46:41 +0200

> Are there any other ops used for block or object layout drivers?

The problem is that I/O is not done using nfs operations in those layout
drivers, but using block or object storage protocols.

I guess the answer is something like:

"If the operation that caused the error was performed using a layout
 specific storage protocol the operation number of the closest ressembling
 NFS operation should be used.

 For the block volume layout type the various READ/WRITE calls in T10 SBC
 should be mapped to READ and WRITE, and SYNCHRONIZE_CACHE or similar
 operations should be mapped to COMMIT.

 For the object layout type <... file in from the OSD people>

 If there is no corresponding NFS operation, de_opnum should be set to
 ILLEGAL.
"

> I think the client should do the mapping.

Absolutely!  There is no need to introduce another set of errors, and
it's rather sad that RFC5664 went down that route.

---

From: Benny Halevy <[email-masked]>
Date: Mon, 19 May 2014 10:36:59 +0300

> +++ b/dotx.d/layoutstats_args.x

Flex-files had an ios_duration field to relate <count, bytes> with a time span.
Otherwise the server's throughput estimate will be rough and possibly inaccurate
based on the time since last report, and this will miss bursts.

Benny