Subject: [nfsv4] revisit space reservations, introduce ALLOCATE operation, V2

From: Christoph Hellwig <[email-masked]>
Date: Thu, 17 Apr 2014 23:09:50 +0200

This is the second updated version of my proposal to replace the
file-wide space reservation with a new ALLOCATE operation.  The
Posix posix_fallocate interface implemented by most Unix systems
allows application to request a range-based reservation, which this
proposal implements.  As it is a true superset of the old file-wide
reservations those have been removed.

Unlike the previous proposal I have not integrated the hole punching
infrastructure and the di_allocated/d_allocated flags into a single
coherent model yet.  I will have a proposal for that soon, which requires
the ALLOCATE operation, but given that the reverse is not true I'd like
to keep it separate for now.

diff --git a/Makefile b/Makefile
index 2efa368..796356a 100755
--- a/Makefile
+++ b/Makefile
@@ -270,6 +270,8 @@ AUTOGEN =	\
 		$(SPITGENXML) \
 		autogen/access_args.xml \
 		autogen/access_res.xml \
+		autogen/allocate_args.xml \
+		autogen/allocate_res.xml \
 		autogen/backchannel_ctl_args.xml \
 		autogen/backchannel_ctl_res.xml \
 		autogen/bind_conn_to_session_args.xml \
@@ -436,6 +438,7 @@ IDXMLSRC_BASE = \
         ${DOC_PREFIX}_middle_fileattributes.xml \
 	${DOC_PREFIX}_middle_op_mandlist.xml \
 	${DOC_PREFIX}_middle_op_aaa.xml \
+	${DOC_PREFIX}_middle_op_allocate.xml \
 	${DOC_PREFIX}_middle_op_copy.xml \
 	${DOC_PREFIX}_middle_op_offload_abort.xml \
 	${DOC_PREFIX}_middle_op_copy_notify.xml \
diff --git a/Makefile.old b/Makefile.old
index 0bb111d..5e60137 100755
--- a/Makefile.old
+++ b/Makefile.old
@@ -276,6 +276,8 @@ AUTOGEN =	\
 		$(SPITGENXML) \
 		autogen/access_args.xml \
 		autogen/access_res.xml \
+		autogen/allocate_args.xml \
+		autogen/allocate_res.xml \
 		autogen/backchannel_ctl_args.xml \
 		autogen/backchannel_ctl_res.xml \
 		autogen/bind_conn_to_session_args.xml \
@@ -440,6 +442,7 @@ IDXMLSRC_BASE = \
         ${DOC_PREFIX}_middle_fileattributes.xml \
 	${DOC_PREFIX}_middle_op_mandlist.xml \
 	${DOC_PREFIX}_middle_op_aaa.xml \
+	${DOC_PREFIX}_middle_op_allocate.xml \
 	${DOC_PREFIX}_middle_op_copy.xml \
 	${DOC_PREFIX}_middle_op_offload_abort.xml \
 	${DOC_PREFIX}_middle_op_copy_notify.xml \
diff --git a/dotx.d/allocate_args.x b/dotx.d/allocate_args.x
new file mode 100644
index 0000000..3f82df5
--- /dev/null
+++ b/dotx.d/allocate_args.x
@@ -0,0 +1,6 @@
+struct ALLOCATE4args {
+        /* CURRENT_FH: file */
+        stateid4        aa_stateid;
+        offset4         aa_offset;
+        length4         aa_length;
+};
diff --git a/dotx.d/allocate_res.x b/dotx.d/allocate_res.x
new file mode 100644
index 0000000..1c0e8df
--- /dev/null
+++ b/dotx.d/allocate_res.x
@@ -0,0 +1,3 @@
+struct ALLOCATE4res {
+	nfsstat4		ar_status;
+};
diff --git a/dotx.d/spit_types.sh b/dotx.d/spit_types.sh
index 8e3b227..b2d7688 100755
--- a/dotx.d/spit_types.sh
+++ b/dotx.d/spit_types.sh
@@ -1004,6 +1004,7 @@ enum nfs_opnum4 {
  OP_SEEK		= 66,
  OP_IO_ADVISE		= 67,
  OP_WRITE_SAME		= 68,
+ OP_ALLOCATE		= 69,
  OP_ILLEGAL		= 10044
 };
 EOF
@@ -1125,6 +1126,7 @@ union nfs_argop4 switch (nfs_opnum4 argop) {
  case OP_SEEK:		SEEK4args opseek;
  case OP_IO_ADVISE:	IO_ADVISE4args opio_advise;
  case OP_WRITE_SAME:	WRITE_SAME4args opwrite_same;
+ case OP_ALLOCATE:	ALLOCATE4args opallocate;
 
  /* Operations not new to NFSv4.1 */
  case OP_ILLEGAL:	void;
@@ -1256,6 +1258,7 @@ union nfs_resop4 switch (nfs_opnum4 resop) {
  case OP_SEEK:		SEEK4res opseek;
  case OP_IO_ADVISE:	IO_ADVISE4res opio_advise;
  case OP_WRITE_SAME:	WRITE_SAME4res opwrite_same;
+ case OP_ALLOCATE:	ALLOCATE4res opallocate_plus;
 
  /* Operations not new to NFSv4.1 */
  case OP_ILLEGAL:	ILLEGAL4res opillegal;
diff --git a/nfsv42_middle_errors.xml b/nfsv42_middle_errors.xml
index 495b0a6..65a65e1 100644
--- a/nfsv42_middle_errors.xml
+++ b/nfsv42_middle_errors.xml
@@ -1334,6 +1334,41 @@
         NFS4ERR_TOO_MANY_OPS
       </c>
 
+      <c>ALLOCATE</c>
+      <c>
+        NFS4ERR_ACCESS,
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_DQUOT,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FBIG,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_IO,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOSPC,
+        NFS4ERR_OLD_STATEID,
+        NFS4ERR_OPENMODE,
+        NFS4ERR_OP_NOT_IN_SESSION,
+        NFS4ERR_REP_TOO_BIG,
+        NFS4ERR_REP_TOO_BIG_TO_CACHE,
+        NFS4ERR_REQ_TOO_BIG,
+        NFS4ERR_RETRY_UNCACHED_REP,
+        NFS4ERR_ROFS,
+        NFS4ERR_SERVERFAULT,
+        NFS4ERR_STALE,
+        NFS4ERR_SYMLINK,
+        NFS4ERR_TOO_MANY_OPS,
+        NFS4ERR_WRONG_TYPE
+      </c>
+
       <c>BACKCHANNEL_CTL</c>
       <c>
         NFS4ERR_BADXDR,
diff --git a/nfsv42_middle_fileattributes.xml b/nfsv42_middle_fileattributes.xml
index 444f680..0c1b6c5 100644
--- a/nfsv42_middle_fileattributes.xml
+++ b/nfsv42_middle_fileattributes.xml
@@ -215,66 +215,11 @@ typedef uint32_t  policy4;
 
     <section toc='exclude' anchor="ss:fattr:sr" title="Attribute 77: space_reserved">
       <t>
-        The space_reserve attribute is a read/write attribute of type
-        boolean.  It is a per file attribute and applies during the
-        lifetime of the file or until it is turned off.  When the space_reserved
-        attribute is set via SETATTR, the server must ensure that there is
-        disk space to accommodate every byte in the file before it can return
-        success.  If the server cannot guarantee this, it must return
-        NFS4ERR_NOSPC.
+        The space_reserved attribute is a read-only attribute per file.
+        If it is true if there are any space reservations on the file, or false
+        otherwise.
       </t>
 
-      <t>
-        If the client tries to grow a file which has the space_reserved
-        attribute set, the server must guarantee that there is disk space to
-        accommodate every byte in the file with the new size before it can
-        return success.  If the server cannot guarantee this, it must return
-        NFS4ERR_NOSPC.
-      </t>
-
-      <t>
-        It is not required that the server allocate the space to the file
-        before returning success.  The allocation can be deferred, however,
-        it must be guaranteed that it will not fail for lack of space.
-      </t>
-
-      <t>
-        The value of space_reserved can be obtained at any time through
-        GETATTR. If the size is retrieved at the same time, the client
-        can determine the size of the reservation.
-      </t>
-
-      <t>
-        In order to avoid ambiguity, the space_reserve bit cannot be set
-        along with the size bit in SETATTR.  Increasing the size of a file
-        with space_reserve set will fail if space reservation cannot be
-        guaranteed for the new size.  If the file size is decreased, space
-        reservation is only guaranteed for the new size.
-        If a hole is punched into the file, then the reservation is not
-        changed.
-      </t>
-
-      <section toc='exclude' anchor='ss:fattr:sr:ex' title='Example of Space Reservation'>
-        <t>
-          If an application used posix_fallocate (<xref target='posix_fallocate' />)
-          to establish a space reservation on a file of size 2G, it might send
-          a compound like this:
-        </t>
-
-        <figure>
-          <artwork>
-  SETATTR (space_reserved = true)
-  SETATTR (size = 2G)
-          </artwork>
-        </figure>
-
-        <t>
-          Note that the ordering of the two SETATTRs does not matter, the second
-          one will return NFS4ERR_NOSPC if the initial guarantee cannot be met.
-        </t>
-      </section>
-    </section>
-
     <section toc='exclude' anchor="ss:fattr:sf" title="Attribute 78: space_freed">
       <t>
         space_freed gives the number of bytes freed if the file is deleted.
diff --git a/nfsv42_middle_introduction.xml b/nfsv42_middle_introduction.xml
index 72ce829..cb92ec7 100644
--- a/nfsv42_middle_introduction.xml
+++ b/nfsv42_middle_introduction.xml
@@ -114,8 +114,8 @@
       <t>
         When a file is sparse, one concern applications have is ensuring
         that there will always be enough data blocks available for the
-        file during future writes. A new attribute, space_reserved
-        (see <xref target='ss:fattr:sr' />) provides the client
+        file during future writes. ALLOCATE
+	(see <xref target='op:allocate' />) allows a client to request
         a guarantee that space will be available.
       </t>
     </section>
diff --git a/nfsv42_middle_op_allocate.xml b/nfsv42_middle_op_allocate.xml
new file mode 100644
index 0000000..c09e60f
--- /dev/null
+++ b/nfsv42_middle_op_allocate.xml
@@ -0,0 +1,50 @@
+<!-- Copyright (C) The IETF Trust (2013) -->
+<!-- Copyright (C) The Internet Society (2013) -->
+
+<section anchor='op:allocate' title='Operation 68: ALLOCATE'>
+  <section toc='exclude' title="ARGUMENT">
+    <?rfc include='autogen/allocate_args.xml'?>
+  </section>
+
+  <section toc='exclude' title="RESULT">
+    <?rfc include='autogen/allocate_res.xml'?>
+  </section>
+
+  <section toc='exclude' title="DESCRIPTION">
+    <t>
+      Whenever a client wishes to reserve space for a region in a file it calls
+      the ALLOCATE operation with the current filehandle set to the filehandle
+      of the file in question, and the start offset and length in bytes of
+      the region set in aa_offset and aa_length respectively.
+    </t>
+
+    <t>
+      The server will ensure that backing blocks are reserved to the region
+      specified by aa_offset and aa_length, and that no future writes into
+      this region will return NFS4ERR_NOSPC. If the region lies partially or
+      fully outside the current file size the file size will be set to
+      aa_offset + aa_length implicitly. If the server cannot guarantee this,
+      it must return NFS4ERR_NOSPC.
+    </t>
+
+    </t>
+      The ALLOCATE operation can also be used to extend the size of a file
+      if the region specified by aa_offset and aa_length extends beyond the
+      current file size.  In that case any data outside of the previous
+      file size will return zeroes when read before data is written to it.
+    </t>
+
+    <t>
+      It is not required that the server allocate the space to the file
+      before returning success.  The allocation can be deferred, however,
+      it must be guaranteed that it will not fail for lack of space.
+    </t>
+
+    <t>
+      The ALLOCATE operation will result in the space_used attribute and
+      space_freed attributes being increased by the number of bytes reserved
+      unless they were previously reserved or written and not shared.
+    </t>
+
+  </section>
+</section>
diff --git a/nfsv42_middle_space.xml b/nfsv42_middle_space.xml
index 701b6ff..3645d33 100644
--- a/nfsv42_middle_space.xml
+++ b/nfsv42_middle_space.xml
@@ -4,21 +4,28 @@
 <section anchor="sec:space" title="Space Reservation">
   <section anchor="ss:space:intro" title="Introduction">
     <t>
-      Applications such as hypervisors want to be able to
-      reserve space for a file, report the amount of
-      actual disk space a file occupies, and free-up the backing space of a
-      file when it is not required.  In virtualized environments, virtual
-      disk files are often stored on NFS mounted volumes.  Since virtual
-      disk files represent the hard disks of virtual machines, hypervisors
-      often have to guarantee certain properties for the file.
+      Applications want to be able to reserve space for a file, report the
+      amount of actual disk space a file occupies, and free-up the backing
+      space of a file when it is not required.
     </t>
 
     <t>
-      One such example is space reservation.  When a hypervisor creates a
-      virtual disk file, it often tries to preallocate the space for the
-      file so that there are no future allocation related errors during the
-      operation of the virtual machine.  Such errors prevent a virtual
-      machine from continuing execution and result in downtime.
+      One example is the posix_fallocate (<xref target='posix_fallocate' />)
+      which allows applications to ask for space reservations from the
+      operating system, usually to provide a better file layout and reduce
+      overhead for random or slow growing file appending workloads.
+    </t>
+
+    <t>
+      Another example is space reservation for virtual disks in a hypervisor.
+      In virtualized environments, virtual disk files are often stored on NFS
+      mounted volumes.  
+
+      When a hypervisor creates a virtual disk file, it often tries to
+      preallocate the space for the file so that there are no future
+      allocation related errors during the operation of the virtual machine.
+      Such errors prevent a virtual machine from continuing execution and
+      result in downtime.
     </t>
 
     <t>
@@ -32,13 +39,10 @@
     </t>
 
     <t>
-      We define a "reservation" as being the combination of the 
-      space_reserved attribute (see <xref target='ss:fattr:sr' />) and
-      the size attribute (see Section 5.8.1.5 of <xref target="RFC5661" />).
-      If space_reserved attribute is
-      set on a file, it is guaranteed that writes that do not grow the file
-      past the size will not fail with NFS4ERR_NOSPC. Once the size is
-      changed, then the reservation is changed to that new size.
+      The new ALLOCATE operation (see <xref target='op:allocate' />)
+      allows a client to request a guarantee that space will be available.
+      The ALLOCATE operation gurantees that any future writes to the
+      region it was successfully called for will not fail with NFS4ERR_NOSPC.
     </t>
 
     <t>
@@ -87,8 +91,8 @@
     <t>
       <list style="hanging">
         <t hangText="space_reserved">
-          This attribute specifies that writes to the reserved
-          area of the file will not fail with NFS4ERR_NOSPACE.
+          This attribute specifies that there are some space reservations
+          on a file.
         </t>
 
         <t hangText="space_freed">

---

From: Tom Haynes <[email-masked]>
Date: Thu, 17 Apr 2014 14:26:29 -0700

It isn’t clear here how a client can use the space_reserved flag
to determine where a space reservation is in effect on a file.

Some nits inline.

> +++ b/dotx.d/allocate_args.x

Since it is an optional operation, you need NFS4ERR_NOTSUPP.

Speaking of which, there should be a corresponding change in nfsv42_middle_op_mandlist.xml.

> +++ b/nfsv42_middle_introduction.xml

Should mention that it sets the space_reserved attribute.

> +++ b/nfsv42_middle_space.xml

guarantees ^^

> _______________________________________________
> nfsv4 mailing list
> [email-masked]
> https://www.ietf.org/mailman/listinfo/nfsv4

---

From: Christoph Hellwig <[email-masked]>
Date: Thu, 17 Apr 2014 23:33:17 +0200

> to determine where a space reservation is in effect on a file.

You can't use it for that fact.  In fact on the Posix side the
reservation model is very simple and doesn't allow any querying at all.

For NFS I actually do eventually plan to offer a query, but that will
be part of a much bigger rework of how the di_allocated / d_allocated
flags work to brings them into a single coherent reservation model
instead of the current reservations and "allocations" that are similar
but still different.  But given that this is a change that affects
larger parts of the spec I'd like to keep it separate.

> Speaking of which, there should be a corresponding change in nfsv42_middle_op_mandlist.xml.

Oķ.

> Should mention that it sets the space_reserved attribute.

Indeed!

> > +      The ALLOCATE operation gurantees that any future writes to the

Thanks, I'll fix this for the next spin.

---

From: Tom Haynes <[email-masked]>
Date: Thu, 24 Apr 2014 10:37:55 -0700

> larger parts of the spec I'd like to keep it separate.

Okay, I’ve taken this patch and applied it to the repo at GitHub. I’ve fixed
up the nits, but I haven’t touched the space_reserved flag.

---

From: Christoph Hellwig <[email-masked]>
Date: Thu, 24 Apr 2014 20:20:05 +0200

> up the nits, but I haven’t touched the space_reserved flag.

Thanks Tom,

I'll post a patch for my ideas about reporting reserved space soon now
that all the other pending changes are in.