I've been kind of struggling with the same thing. On a production cluster under my responsibility I found a total of 365 unassociated objects and wrote a Python script to check them with ease. It is a VDI cluster so I suspect a some kind of race between App Volumes and Horizon when it comes to deleting an ephemeral VM and its associated disks considering also Appvolumes REDO vmdks. Least that's my guess for the possible cause.
Two cases were easy to delete. First was empty VM namespaces i.e. a VM namespace object with only VMFS metadata files present. The other was vdisk objects when there was not anymore corresponding osfs file present. The former cases I deleted by calling osfs-rmdir and latter with objtool delete. 222/365 of objects were either of these and I got easily rid of. However the cases that remained were namespaces with files and vdisks that still had their osfs file association present. I couldn't figure out what would be the correct action or order of actions to delete these.
The script is this:
#!/usr/bin/env python
'''
Remove vsan orphaned objects with rudimentary intelligence and user control
First argument is the list to process
Author: perttu.maatta@h*******.fi
Public domain
Script assumes following the vsanobj metadata
{
"UUID":"daefc660-60db-5079-a9a8-98f2b3da1830",
"Object type":"vsan",
"Object size":"273804165120",
"Allocation type":"Thin",
"Policy":"((\"stripeWidth\" i1) (\"cacheReservation\" i0) (\"proportionalCapacity\" (i0 i100)) (\"hostFailuresToTolerate\" i1) (\"spbmProfileId\" \"818c0891-b8cc-45aa-9ab9-d1e070c3e68e\") (\"spbmProfileGenerationNumber\" l+0) (\"objectVersion\" i15) (\"CSN\" l1258) (\"SCSN\" l1247) (\"spbmProfileName\" \"VM_HOME_6984ae27-1af1-4bcf-a58b-7218dfcf6982\"))",
"Object class":"vmnamespace",
"Object capabilities":"NONE",
"Object path":"/vmfs/volumes/vsan:521bea82245708eb-f874cefcb89cf014/vdi-win1",
"Group uuid":"00000000-0000-0000-0000-000000000000",
"Container uuid":"00000000-0000-0000-0000-000000000000",
"IsSparse":"0",
"User friendly name":"vdi-win1",
"HA metadata":"(null)",
"Filesystem type":"vmfs6d"
}
'''
import sys
import subprocess as subp
import json
from datetime import datetime
from pathlib import Path
OSFSPATH = '/usr/lib/vmware/osfs/bin'
OBJTOOL = '/'.join([OSFSPATH,'objtool'])
LS = '/'.join([OSFSPATH,'osfs-ls'])
RMDIR = '/'.join([OSFSPATH,'osfs-rmdir'])
VMFSMETADATA = {'.fbb.sf', '.fdc.sf', '.pbc.sf', '.sbc.sf', '.vh.sf', '.pb2.sf', '.sdd.sf', '.jbc.sf', ''}
logfile = None
def logAction(jsonObj, isEmpty, exists, isDeleted):
# header: uuid, class, path, empty(VM namespaces), exists in filesystem (vDisk), deleted
datarow = [ jsonObj['UUID'], jsonObj['Object class'], jsonObj['Object path'], str(isEmpty), str(exists), str(isDeleted) ]
if logfile is not None:
logfile.write(','.join(datarow)+'\n')
def pUuid(uuid):
p = subp.Popen([OBJTOOL,'getAttr','-u',uuid,'--format','JSON'],stdout=subp.PIPE,stderr=subp.PIPE)
stdout, stderr = p.communicate()
if p.returncode > 0:
return
jsonObj = json.loads(stdout)
if jsonObj['Object class'] == 'vmnamespace':
pVMnamespace(jsonObj)
elif jsonObj['Object class'] == 'vdisk':
pVdisk(jsonObj)
else:
print(f"skipping object class {jsonObj['Object class']}")
logAction(jsonObj,'','',False)
# processes VMnamespace type objects
def pVMnamespace(jsonObj):
# checking directory contents
path = jsonObj['Object path']
p = subp.Popen([LS, path],stdout=subp.PIPE,stderr=subp.PIPE)
stdout, stderr = p.communicate()
items = set(stdout.decode('utf8').split('\n'))
print("number of files and folders (including vmfs-metadata) in %s: %d" % (path, (len(items)-1)))
leftovers= list(items - VMFSMETADATA)
if len(leftovers) == 0:
print("%s has only VMFS-metadata" % path)
isEmpty = True
if question():
rmdir(path)
isDeleted = True
else:
print("data files: %s" % " ".join(leftovers))
isEmpty = False
isDeleted = False
logAction(jsonObj, isEmpty, '', isDeleted)
# processes vDisk type objects
def pVdisk(jsonObj):
if Path(jsonObj['Object path']).exists():
# if object exists as a file in a VM namespace, what is the correct delete action then?
print(f"object {jsonObj['Object path']} exists as accessible osfs file, lets skip it")
logAction(jsonObj, '', True, False)
return
print(f"object {jsonObj['Object path']} doesn't exist as an accessible osfs file")
isDeleted = False
if question():
rmObject(jsonObj['UUID'])
isDeleted = True
logAction(jsonObj, '', False, isDeleted)
def question():
dObj = None
while(dObj != "y" and dObj != "n"):
dObj = input("delete object [y|n]")
return dObj == "y"
def rmdir(dir):
subp.run([RMDIR,dir])
def rmObject(uuid):
p = subp.Popen([OBJTOOL,'delete','-u',uuid],stdout=subp.PIPE,stderr=subp.PIPE)
stdout, stderr = p.communicate()
print(f"stdout:{stdout.decode()}\nstderr: {stderr.decode()}")
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit(1)
logprefix = datetime.today().strftime('%Y-%m-%d-%H-%M-%S')
logfile = open(logprefix + '-vsancleanup.log', 'w')
logfile.write('uuid, class, path, empty(VM namespaces), exists in filesystem (vDisk), deleted\n')
filename = sys.argv[1]
file = open(filename,'r')
for l in file:
print("handling uuid %s" % l, end="")
pUuid(l)
logfile.close()
