I did a lot of research on table storage and size limits, overhead, etc., using these sources:
Using this information, I wrote code to efficiently store binary data on several properties, calculating any overhead information about strings and properties and staying within the property limit of 64 KB and the line limit of 1 MB.
Unfortunately, this just doesn't work. As an example of storing about 0.5 MB, 400 Bad Request is returned, indicating that the object is too large, and I do not understand why it will be given a 1MB row size limit.
<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<error xmlns="http://schemas.microsoft.com/ado/2007/08/dataservices/metadata">
<code>EntityTooLarge</code>
<message xml:lang="en-GB">The entity is larger than allowed by the Table Service.</message>
</error>
The code I used is pretty straight forward, but I could be wrong in estimating the overhead - however, I doubt that it will be turned off by 100% of the data size.
class Program
{
static void Main(string[] args)
{
var client = CloudStorageAccount.DevelopmentStorageAccount.CreateCloudTableClient();
var table = client.GetTableReference("sometable");
table.CreateIfNotExists();
const int rowOverhead = 4;
const int maxRowSize = 1024 * 1024;
const int maxProperties = 252;
const int maxPropertySize = 64 * 1024;
var stream = new MemoryStream(new byte[512 * 1024]);
var entity = new DynamicTableEntity("pk", "rk");
var buffer = new byte[maxPropertySize];
var keySize = (entity.PartitionKey.Length + entity.RowKey.Length) * 2;
var used = rowOverhead + keySize;
for (var i = 0; i < maxProperties + 1; i++)
{
if (i > maxProperties)
{
throw new ArgumentException(string.Format("You have exceeded the column limit of {0}.", maxProperties));
}
var name = string.Concat("d", i);
var overhead = CalculatePropertyOverhead(name, EdmType.Binary);
var read = stream.Read(buffer, 0, maxPropertySize - overhead);
used += read + overhead;
if (used > maxRowSize)
{
throw new ArgumentException(string.Format("You have exceeded the max row size of {0} bytes.", maxRowSize));
}
if (read > 0)
{
var data = new byte[read];
Array.Copy(buffer, 0, data, 0, read);
entity.Properties.Add(name, new EntityProperty(data));
}
else
{
break;
}
}
Console.WriteLine("Total entity size: {0}", used);
table.Execute(TableOperation.InsertOrReplace(entity));
}
static int CalculatePropertyOverhead(string name, EdmType type)
{
const int propertyOverhead = 8;
int propertyNameSize = name.Length * 2;
int propertyTypeSize;
switch (type)
{
case EdmType.Binary:
case EdmType.Int32:
case EdmType.String:
propertyTypeSize = 4;
break;
case EdmType.Boolean:
propertyTypeSize = 1;
break;
case EdmType.DateTime:
case EdmType.Double:
case EdmType.Int64:
propertyTypeSize = 8;
break;
case EdmType.Guid:
propertyTypeSize = 16;
break;
default:
throw new NotSupportedException();
}
return propertyOverhead + propertyNameSize + propertyTypeSize;
}
}
Any help in explaining what I am missing is appreciated!
Thank,
Matthias